Skip to content

Automate browser based workflows with AI

Return the number a user asked for, not the one beside it (SKY-13226) (#8029)

Andrew Neilson2401cbcf15d4
diff --git a/skyvern/config.py b/skyvern/config.py
index 1c527323a..4fce14b41 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -207,9 +207,10 @@ class Settings(BaseSettings):
COPILOT_REQUEST_POLICY_CLASSIFIER_TIMEOUT_SECONDS: float = 12.0
COPILOT_TURN_INTENT_CLASSIFIER_TIMEOUT_SECONDS: float = 12.0
COPILOT_COMPLETION_JUDGE_TIMEOUT_SECONDS: float = 12.0
- # Consecutive repair runs that make no newly-verified forward progress before the
- # copilot stops re-running and escalates honestly. Set very high to disable the ceiling.
- COPILOT_SCOUT_ACT_OBSERVE_TIMEOUT_SECONDS: float = 4.0
+ # A capture that runs out of time yields no page evidence at all, so the scout falls back to
+ # dumping the page. Measured attaches on a live dashboard reach 8.6s; the former 4s bound cut the
+ # tail off and a third of captures returned nothing.
+ COPILOT_SCOUT_ACT_OBSERVE_TIMEOUT_SECONDS: float = 12.0
# Bounded settle-then-re-perceive after a non-advancing click on a precondition-gated control:
# re-probe the side-effect-free extractor a few times (hard-capped) until a just-issued AJAX populates.
COPILOT_CLICK_SETTLE_MAX_PROBES: int = 3
diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2
index a9fb41af4..4f7495cd0 100644
--- a/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2
+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2
@@ -104,6 +104,7 @@ completion_criteria:
- judgment_predicate: null unless expected_output_shape=goal_judgment_boolean and requested_output_evidence_source=independent_run_evidence. Use only the closed-vocabulary value login_gate_blocks_target, and only when the requested judgment is whether login gating blocks the target path.
- judgment_polarity_when_holds: null unless judgment_predicate is non-null. Set it to the boolean output value that means the predicate holds; for login_gate_blocks_target this is true.
- requested_output_evidence_source: runtime_output|independent_run_evidence|registered_output_parameter|registered_artifact_content. Use runtime_output for site-produced values the code block directly reads from the page or browser state, such as dates, amounts, statuses, identifiers, names, or downloaded-file descriptors. Use independent_run_evidence only for artifact-computed judgments, selections, superlatives, rankings, validation booleans, or "best/correct/cheapest/highest" claims that cannot prove themselves by returning their own answer. Use registered_output_parameter or registered_artifact_content only when the requested datum must be proven by execution-owned registered output parameters or registered artifact content. Do not choose this from output_path names, fixture names, domain keywords, or prose labels; choose it from the user's requested evidence contract.
+- output_path: set output.<field> whenever the user asks the workflow to return, report, output, extract, or tell them a value, including a plain single value such as a count, total, price, date, status, or name. The only valid form is output.<field>. Leave it null only when the criterion's end state is an action or a side effect rather than a value handed back to the user. A request phrased around a metric rather than a field name still returns a value: name the field after the value the user asked for.
- When a selection, priority, ranking, or superlative criterion governs a returned requested-output field, set output_path for that returned field and requested_output_evidence_source=independent_run_evidence; the selected value cannot prove its own correctness.
- Example: if the user asks to return the selected or highest-priority document name, emit the criterion with output_path=output.document_name and requested_output_evidence_source=independent_run_evidence.
- kind=outcome|terminal_action|validation_classification. Use validation_classification only when the user's no-side-effect goal is itself a judgment/classification returned by the workflow, such as whether a path is login-gated. Use terminal_action only when the criterion's end state is a terminal action in the external site for one of the supported families below. Use outcome for all other outcomes.
diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-request-slots.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-request-slots.j2
index cf27dc61e..904b4df1a 100644
--- a/skyvern/forge/prompts/skyvern/workflow-copilot-request-slots.j2
+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-request-slots.j2
@@ -8,6 +8,7 @@ Schema-valid examples:
{"version":"1","slots":[{"source_id":"u0","source_quote":"report the blocker if online submission is unavailable","plane":"run","pinability":"shapeless_valid","antecedent_family":"blocker"}],"datum_bindings":[]}
{"version":"1","slots":[{"source_id":"u0","source_quote":"request id","plane":"run","pinability":"shapeless_valid","antecedent_family":"unconditional"}],"datum_bindings":[{"criterion_index":0,"datum_field":"output_path","declined":false,"source_id":"u0","source_quote":"request id"}]}
{"version":"1","slots":[],"datum_bindings":[{"criterion_index":0,"datum_field":"output_path","declined":true}]}
+{"version":"1","slots":[],"datum_bindings":[],"candidate_refutations":[{"candidate":"visitors","reason":"names a page section, not a value the user asked the workflow to return"}]}
Rules:
- Return version exactly "1". Return an empty slots list when the user requests no output member; empty membership is an explicit contract, not an error.
@@ -28,6 +29,7 @@ Rules:
- Do not infer slots from sampled page shapes or non-user context. Do not normalize, rename, or paraphrase source quotes.
- Return datum_bindings as an array. When DATUM TARGETS is empty, return an empty array. Otherwise answer every target exactly once and no others, identifying each target only by its criterion_index and datum_field. When user text names the datum, return exactly {"criterion_index":<target integer>,"datum_field":<target field>,"declined":false,"source_id":<source ID>,"source_quote":<smallest unique exact quote>} whose independently declared slot names that target datum. Every accepted binding must name one exact slot in your slots array, and two targets may not bind the same slot. When no source quote actually names a target datum, return exactly {"criterion_index":<target integer>,"datum_field":<target field>,"declined":true}. The server owns datum_value and criterion_outcome_sha256; never return either field. Decline instead of guessing or borrowing an adjacent datum.
- When a target's datum_value leaf directly matches a `whether ...` output phrase (underscores are spaces; articles and forms of "be" are insignificant), declare and bind that exact full `whether ...` phrase. Never bind the target to an earlier, broader task goal or evidence-gathering instruction.
+- DETECTED OUTPUT CANDIDATES lists field names a deterministic scan found requested in the user sources. Answer every candidate exactly once: either one of your slots' source_quote contains the candidate text, or return {"candidate":<name>,"reason":<short reason>} in candidate_refutations. An envelope silent on a candidate is invalid. A refutation is a real judgment that the user did not ask for that value, not a shortcut; when the sources do request the value, declare the slot.
Ordered, deduplicated user request sources:
```{{ request_sources }}```
@@ -39,3 +41,5 @@ Bounded global context (non-authoritative):
```{{ global_context }}```
Datum targets from the immutable request-policy payload:
```{{ datum_targets }}```
+Detected output candidates from the deterministic request scan:
+```{{ detected_output_candidates }}```
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 29d5c4741..92cf0780e 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -74,7 +74,6 @@ from skyvern.forge.sdk.copilot.completion_criteria_store import (
apply_requested_output_producer_floor,
build_turn_state,
reconcile_completion_criteria,
- requested_output_paths,
)
from skyvern.forge.sdk.copilot.completion_verification import only_structural_requested_output_abstentions
from skyvern.forge.sdk.copilot.config import (
@@ -114,11 +113,15 @@ from skyvern.forge.sdk.copilot.credential_pause import credential_pause_reason,
from skyvern.forge.sdk.copilot.data_write_defaults import default_data_write_continue_on_failure
from skyvern.forge.sdk.copilot.enforcement import (
artifact_health_blocked,
+ dump_derivation_inputs,
log_scouted_spine_unresolved_at_turn_halt,
outcome_fully_verified,
recycle_admits_present_completion_contract_ask,
requested_output_extraction_plan,
requested_output_extraction_plan_changed,
+ requested_output_extraction_plan_diagnostic,
+ requested_output_paths_for_derivation,
+ synthesized_offer_reopened_for_extraction_plan,
synthesized_persistence_reopened,
synthesized_persistence_reopened_after_failed_run,
synthesized_trajectory_is_goal_complete,
@@ -135,6 +138,7 @@ from skyvern.forge.sdk.copilot.output_contracts import (
OUTPUT_CONTRACT_ACTUATION_EXHAUSTED_REASON_CODE,
OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE,
)
+from skyvern.forge.sdk.copilot.output_extraction_plan import unbound_candidate_relations
from skyvern.forge.sdk.copilot.output_policy import (
ACTUATION_OBLIGATION_STEER_REASON_CODE,
ACTUATION_OBLIGATION_UNMET_REASON_CODE,
@@ -1173,6 +1177,29 @@ def _recorded_build_test_outcome_prompt(ctx: CopilotContext | None) -> str:
return "\n\n" + "\n".join(line for line in lines if line)
+def _unbound_output_read_offer(ctx: CopilotContext, extraction_plan: object) -> str:
+ """What the page offers for a requested output its own vocabulary never names.
+
+ The extraction binds a minted label to a page label, so a page calling the requested quantity
+ something else binds nothing however plainly it shows it. Naming what the page does offer lets
+ the read that follows be the join: reading one into the requested path lets the value witness
+ bind the relation still showing that value, on the quantity rather than on the wording.
+ """
+ if extraction_plan is not None:
+ return ""
+ paths = sorted(requested_output_paths_for_derivation(ctx))
+ candidates = unbound_candidate_relations(ctx.flow_evidence)
+ if not paths or not candidates:
+ return ""
+ offered = "\n".join(f"- {label} = {value}" for label, value in candidates)
+ return (
+ "\n\nThe page does not label any value with the wording this request uses, so the extraction "
+ f"cannot bind {', '.join(paths)} yet. The page is currently showing:\n{offered}\n"
+ "If one of these is the requested quantity, read it with read_value and claim the requested "
+ "output path, and the extraction will bind to the value you observed rather than to a word."
+ )
+
+
def _synthesized_block_offer_prompt(ctx: CopilotContext | None) -> str:
"""Pre-authoring offer of the synthesized code block.
@@ -1185,7 +1212,13 @@ def _synthesized_block_offer_prompt(ctx: CopilotContext | None) -> str:
LOG.debug("copilot_synthesized_block_offer_skipped", reason="policy_not_code_only_browser")
return ""
reopened_after_failed_run = synthesized_persistence_reopened_after_failed_run(ctx)
- reopened = synthesized_persistence_reopened(ctx)
+ # Derived before the authored-once gate rather than after it: a page that only reveals the
+ # requested value later makes the read provable after the draft exists, and asking whether the
+ # offer should reopen only once it has already returned means the proven read is never offered.
+ extraction_plan = requested_output_extraction_plan(ctx)
+ reopened = synthesized_persistence_reopened(ctx) or synthesized_offer_reopened_for_extraction_plan(
+ ctx, extraction_plan
+ )
if ctx.update_workflow_called and not reopened:
LOG.debug("copilot_synthesized_block_offer_skipped", reason="already_authored")
return ""
@@ -1195,15 +1228,29 @@ def _synthesized_block_offer_prompt(ctx: CopilotContext | None) -> str:
trajectory_len = len(ctx.scout_trajectory)
previous_offer_len = ctx.synthesized_block_offered_trajectory_len
trajectory_goal_complete = synthesized_trajectory_is_goal_complete(ctx)
- extraction_plan = requested_output_extraction_plan(ctx)
- request_policy = getattr(ctx, "request_policy", None)
- if (
- request_policy is not None
- and requested_output_paths(request_policy.completion_criteria)
- and extraction_plan is None
- ):
- LOG.debug("copilot_synthesized_block_offer_skipped", reason="extraction_plan_unavailable")
- return ""
+ if requested_output_paths_for_derivation(ctx):
+ if extraction_plan is None:
+ # Withholding the offer leaves the turn with no block to run, so a plan that cannot bind
+ # falls back to the spine the scout demonstrated rather than replacing it with nothing.
+ LOG.info(
+ "copilot_synthesized_block_offer_degraded_to_spine",
+ reason="extraction_plan_unavailable",
+ **requested_output_extraction_plan_diagnostic(ctx),
+ )
+ dump_derivation_inputs(ctx, outcome="degraded")
+ else:
+ LOG.info(
+ "copilot_synthesized_block_offer_extraction_plan_derived",
+ requested_output_paths=list(extraction_plan.requested_output_paths),
+ bound_output_paths=[binding.output_path for binding in extraction_plan.live_reads],
+ relation_labels=[binding.relation_label for binding in extraction_plan.live_reads],
+ )
+ dump_derivation_inputs(ctx, outcome="derived")
+ else:
+ # A turn that never forms a requested output still authors and runs code, and its trajectory
+ # is what a login or navigation defect has to be replayed from. Dumping only where a plan was
+ # attempted left exactly those turns unreproducible offline.
+ dump_derivation_inputs(ctx, outcome="no_requested_output")
plan_changed = requested_output_extraction_plan_changed(ctx, extraction_plan)
reopened = reopened or plan_changed
if (
@@ -1213,7 +1260,7 @@ def _synthesized_block_offer_prompt(ctx: CopilotContext | None) -> str:
and not reopened
and not plan_changed
):
- LOG.debug(
+ LOG.info(
"copilot_synthesized_block_offer_skipped",
reason="already_offered",
previous_trajectory_len=previous_offer_len,
@@ -1274,7 +1321,8 @@ def _synthesized_block_offer_prompt(ctx: CopilotContext | None) -> str:
code_len=len(synthesized.code),
)
goal = ctx.block_goal_main_goal or ctx.user_message or ""
- return "\n\n" + render_synthesized_offer_text(synthesized, ctx.scout_trajectory, goal=goal)
+ offer = "\n\n" + render_synthesized_offer_text(synthesized, ctx.scout_trajectory, goal=goal)
+ return offer + _unbound_output_read_offer(ctx, extraction_plan)
def _build_dynamic_system_prompt(tool_usage_guide: str, config: CopilotConfig) -> Callable[[object, object], str]:
@@ -4820,7 +4868,12 @@ async def run_copilot_agent(
ctx_sink[0] if ctx_sink else None, error_result, preserve_result=True
)
finally:
- finalize_outcome_verification_trace(ctx_sink[0] if ctx_sink else None, turn_span)
+ turn_end_ctx = ctx_sink[0] if ctx_sink else None
+ finalize_outcome_verification_trace(turn_end_ctx, turn_span)
+ if turn_end_ctx is not None:
+ # The other dump sites hang off prompt construction, so a turn that ended on the
+ # clock or on an error composed no further prompt and recorded nothing at all.
+ dump_derivation_inputs(turn_end_ctx, outcome="turn_end")
except Exception as exc:
LOG.error(
"Copilot turn unhandled error",
diff --git a/skyvern/forge/sdk/copilot/code_block_synthesis.py b/skyvern/forge/sdk/copilot/code_block_synthesis.py
index 1bb8505a4..5848a2a8c 100644
--- a/skyvern/forge/sdk/copilot/code_block_synthesis.py
+++ b/skyvern/forge/sdk/copilot/code_block_synthesis.py
@@ -54,6 +54,10 @@ LOG = structlog.get_logger()
_MAX_STEPS = 60
_INDENT = " "
+# A dashboard renders the tile before the figure it will hold, so a designated read waits for the
+# value rather than reporting the empty frame it lands in first.
+_DESIGNATED_VALUE_SETTLE_POLLS = 60
+_DESIGNATED_VALUE_SETTLE_INTERVAL_MS = 500
_DOMCONTENTLOADED = "domcontentloaded"
_ENTRY_TARGET_VAR = "_scout_entry_target"
_DOWNLOAD_TARGET_VAR = "_scout_download_target"
@@ -429,7 +433,6 @@ _STRUCTURAL_DISMISSAL_SELECTOR_PATTERN = re.compile(
# container). A wait returns the moment its condition holds, so a fast page pays nothing; only a
# genuinely absent state pays the full budget. Distinct from the deliberate 1s speculative probes.
_REQUIRED_STATE_TIMEOUT_MS = 120_000
-
_SYNTHESIZED_BLOCK_LABEL = "scout_synthesized_browser_steps"
# Names the code-block executor reserves in its exec() namespace (block.py build_safe_vars
@@ -2443,6 +2446,18 @@ def synthesize_code_block(
if compile_download_target and reached_download_target is not None
else ""
)
+ # Which element proves the block is where the flow starts is a separate question from which
+ # step it resumes at: a scout that opened a password-reset link before signing in anchored the
+ # whole login on the link it then clicked away from, while still needing to replay that click.
+ # The anchor prefers a durable target - something the flow fills or selects - and the replay
+ # start is left exactly where it was.
+ durable_anchor_target, _durable_anchor_index = _entry_target_locator(
+ entry_trajectory, strict_selectors=strict_selectors, prefer_durable=True
+ )
+ if fallback_entry_target and durable_anchor_target:
+ # Re-anchoring to the first-touched element after navigating would put the block back on
+ # the link it clicked away from, so the durable target is the anchor on both paths.
+ fallback_entry_target = durable_anchor_target
entry_target = download_entry_target if download_entry_target else fallback_entry_target
entry_replay_condition_active = bool(download_entry_target and fallback_entry_target)
entry_replay_start_index = (
@@ -2486,7 +2501,14 @@ def synthesize_code_block(
for interaction in entry_trajectory
)
)
+ login_guard_last_index: int | None = None
if login_only_presence_guard_active:
+ credential_index = last_scout_credential_fill_index(entry_trajectory)
+ login_guard_last_index = (
+ credential_submit_boundary_index(entry_trajectory, credential_index)
+ if credential_index is not None
+ else None
+ )
notes.append(
"login rung fills only when the credential form is present, so an authenticated replay skips it"
)
@@ -2591,7 +2613,11 @@ def synthesize_code_block(
return _INDENT * 2
if entry_post_auth_resume_index and trajectory_index < entry_post_auth_resume_index:
return _INDENT * 2
- if login_only_presence_guard_active:
+ # The guard exists so an authenticated replay skips the login. Indenting past its submit
+ # would skip the value read too, and the block then returns a name it never bound.
+ if login_only_presence_guard_active and (
+ login_guard_last_index is None or trajectory_index <= login_guard_last_index
+ ):
return _INDENT * 2
return _INDENT
@@ -2691,13 +2717,27 @@ def synthesize_code_block(
continue
line_start = len(lines) + 1
variable = f"_read_value_{len(read_bindings)}"
+ # A read is only recorded once it returns something, so an empty replay contradicts the
+ # proof whatever shape that proof had; an empty collection here is absence, not a correct
+ # answer for a request that legitimately has none.
+ absent = '(None, "", [], {})'
lines.append(f"{action_indent}{variable} = await page.evaluate({expression!r})")
poll_rounds = _REQUIRED_STATE_TIMEOUT_MS // 1000
lines.append(f"{action_indent}for _ in range({poll_rounds}):")
- lines.append(f'{action_indent}{_INDENT}if {variable} is not None and {variable} != "":')
+ lines.append(f"{action_indent}{_INDENT}if {variable} not in {absent}:")
lines.append(f"{action_indent}{_INDENT * 2}break")
lines.append(f"{action_indent}{_INDENT}await page.wait_for_timeout(1000)")
lines.append(f"{action_indent}{_INDENT}{variable} = await page.evaluate({expression!r})")
+ # The scout only records a read that returned something, so an absent replay contradicts
+ # the proof this read was built from. Returning the absent value instead reports success
+ # while the requested field carries nothing.
+ lines.append(f"{action_indent}if {variable} in {absent}:")
+ # Exception is the only type the runtime code sandbox resolves; a narrower one fails the
+ # sandbox name check.
+ lines.append(
+ f"{action_indent}{_INDENT}raise Exception("
+ f"{f'{output_path} was not present on the page: '!r} + {expression!r})"
+ )
read_bindings.append((output_path, variable))
record_emission(trajectory_index, tool_name, "evaluate", "page", line_start=line_start, lane="page_read")
append_step(f"Read {output_path.removeprefix('output.')}", "extract", line_start)
@@ -3450,9 +3490,56 @@ def _array_prefix(binding: LiveReadBinding) -> tuple[tuple[str, bool], ...]:
return _array_prefix_of_segments(output_path_segments(binding.output_path))
+def _scalar_label_proof_statements(binding: LiveReadBinding, children: str) -> list[str]:
+ """Re-read the label that proves this value's identity, wherever it lives.
+
+ A tile that renders its heading and its figure in separate branches has no container holding both,
+ so proving the label only as the value's first sibling makes such a value unreadable. A tile that
+ prints its figure first ("1.22K logs found") keeps the label beside it but not at child zero.
+ """
+ anchor = (
+ f"page.locator({json.dumps(binding.label_selector)})"
+ if binding.label_selector
+ else f"{children}.nth({binding.label_child_index})"
+ )
+ target = f"{anchor}.first" if binding.label_selector else anchor
+ return [
+ f"if not await {target}.is_visible():",
+ f'{_INDENT}raise Exception("Observed scalar label is no longer visible")',
+ f"if (await {target}.inner_text()).strip() != {json.dumps(binding.relation_label)}:",
+ f'{_INDENT}raise Exception("Observed scalar label changed")',
+ ]
+
+
def _key_value_scalar_read_statements(binding: LiveReadBinding, variable: str, *, guard_empty: bool) -> list[str]:
container = f"page.locator({json.dumps(binding.selector)})"
target = f"{container}.nth({binding.selector_index})"
+ if binding.child_count == 0:
+ # A designated read: the model pointed at the value element itself, so the read takes its
+ # text directly — no child-slot walk, and no label pin, since the value is what changes.
+ statements = [
+ f'await {container}.first.wait_for(state="visible", timeout={_REQUIRED_STATE_TIMEOUT_MS})',
+ f"if await {container}.count() != {binding.selector_count}:",
+ f'{_INDENT}raise Exception("Designated value selector cardinality changed")',
+ f"if not await {target}.is_visible():",
+ f'{_INDENT}raise Exception("Designated value element is no longer visible")',
+ # The container renders before the figure does on a page still resolving its query, and
+ # reading once there returns the empty frame the value lands in a moment later.
+ f"{variable} = (await {target}.inner_text()).strip()",
+ f"for _ in range({_DESIGNATED_VALUE_SETTLE_POLLS}):",
+ f"{_INDENT}if {variable}:",
+ f"{_INDENT * 2}break",
+ f"{_INDENT}await page.wait_for_timeout({_DESIGNATED_VALUE_SETTLE_INTERVAL_MS})",
+ f"{_INDENT}{variable} = (await {target}.inner_text()).strip()",
+ ]
+ if guard_empty:
+ statements.extend(
+ [
+ f"if not {variable}:",
+ f'{_INDENT}raise Exception("Designated value element is empty")',
+ ]
+ )
+ return statements
children = f'{target}.locator(":scope > *")'
statements = [
f'await {container}.first.wait_for(state="visible", timeout={_REQUIRED_STATE_TIMEOUT_MS})',
@@ -3462,10 +3549,7 @@ def _key_value_scalar_read_statements(binding: LiveReadBinding, variable: str, *
f'{_INDENT}raise Exception("Observed scalar relation is no longer visible")',
f"if await {children}.count() != {binding.child_count}:",
f'{_INDENT}raise Exception("Observed scalar direct-child shape changed")',
- f"if not await {children}.nth(0).is_visible():",
- f'{_INDENT}raise Exception("Observed scalar label is no longer visible")',
- f"if (await {children}.nth(0).inner_text()).strip() != {json.dumps(binding.relation_label)}:",
- f'{_INDENT}raise Exception("Observed scalar label changed")',
+ *(_scalar_label_proof_statements(binding, children) if binding.identified_by_label else []),
f"if not await {children}.nth({binding.child_index}).is_visible():",
f'{_INDENT}raise Exception("Observed scalar value is no longer visible")',
f"{variable} = (await {children}.nth({binding.child_index}).inner_text()).strip()",
@@ -3813,14 +3897,17 @@ def produce_covered_static_return_envelope(
def _trajectory_contains_reveal(trajectory: Sequence[Mapping[str, Any]], plan: RequestedOutputExtractionPlan) -> bool:
+ if plan.reveal is None:
+ return True
+ reveal = plan.reveal
return any(
str(interaction.get("tool_name") or "") == "click"
and (
- (bool(plan.reveal.selector) and str(interaction.get("selector") or "") == plan.reveal.selector)
+ (bool(reveal.selector) and str(interaction.get("selector") or "") == reveal.selector)
or (
- bool(plan.reveal.role and plan.reveal.name)
- and str(interaction.get("role") or "") == plan.reveal.role
- and str(interaction.get("accessible_name") or "") == plan.reveal.name
+ bool(reveal.role and reveal.name)
+ and str(interaction.get("role") or "") == reveal.role
+ and str(interaction.get("accessible_name") or "") == reveal.name
)
)
for interaction in trajectory
diff --git a/skyvern/forge/sdk/copilot/completion_criteria_store.py b/skyvern/forge/sdk/copilot/completion_criteria_store.py
index 77dc75374..69d70343e 100644
--- a/skyvern/forge/sdk/copilot/completion_criteria_store.py
+++ b/skyvern/forge/sdk/copilot/completion_criteria_store.py
@@ -79,6 +79,10 @@ def _normalize_floor_rekeyed_association(marker: object, path: object) -> tuple[
return False, None
+def _stored_requested_output_label(raw: object) -> str | None:
+ return raw.strip() or None if isinstance(raw, str) else None
+
+
@dataclass(frozen=True)
class StoredCriteriaSet:
set_id: str
@@ -175,6 +179,8 @@ def criteria_to_json(criteria: tuple[CompletionCriterion, ...] | list[Completion
"classification_output_key": criterion.classification_output_key,
"expected_classification": criterion.expected_classification,
}
+ if criterion.requested_output_label is not None:
+ item["requested_output_label"] = criterion.requested_output_label
if criterion.deliverable_confirmation_criterion_id is not None:
item["deliverable_confirmation_criterion_id"] = criterion.deliverable_confirmation_criterion_id
if criterion.kind == "terminal_action":
@@ -379,6 +385,7 @@ def criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:
method_mandated=stored_method_mandated,
level=stored_level, # type: ignore[arg-type]
output_path=stored_output_path,
+ requested_output_label=_stored_requested_output_label(item.get("requested_output_label")),
expected_output_value=stored_expected_output_value,
expected_output_shape=stored_expected_output_shape,
requested_output_evidence_source=cast(RequestedOutputEvidenceSource, requested_output_evidence_source),
diff --git a/skyvern/forge/sdk/copilot/completion_output_grounding.py b/skyvern/forge/sdk/copilot/completion_output_grounding.py
index 14d0a13c3..0e156d243 100644
--- a/skyvern/forge/sdk/copilot/completion_output_grounding.py
+++ b/skyvern/forge/sdk/copilot/completion_output_grounding.py
@@ -269,7 +269,9 @@ def grade_requested_output_criteria(
requires_independent_evidence = (
effective_evidence_source == "independent_run_evidence" or _is_judgment_boolean_criterion(criterion)
)
- if requires_independent_evidence and satisfied_source not in _INDEPENDENT_EVIDENCE_SOURCES:
+ if self_witnessed_numeric_substring(criterion, snapshot, satisfied_source) or (
+ requires_independent_evidence and satisfied_source not in _INDEPENDENT_EVIDENCE_SOURCES
+ ):
verdicts.append(
CriterionVerdict(
criterion_id=criterion.id,
@@ -430,7 +432,7 @@ def _carrier_confirmation(
artifact_text = _carrier_surface_text(
snapshot, _REGISTERED_ARTIFACT_OBSERVATION_LABEL, "registered_artifact_content"
)
- if artifact_text is not None and _boundary_delimited_present(value_text, artifact_text):
+ if artifact_text is not None and _value_present_in_text(value_text, artifact_text):
return CriterionVerdict(
criterion_id=criterion.id,
state="satisfied",
@@ -440,11 +442,11 @@ def _carrier_confirmation(
**trace_fields,
)
page_text = _carrier_surface_text(snapshot, _POST_RUN_PAGE_OBSERVATION_LABEL, "independent_page_evidence")
- if page_text is not None and _boundary_delimited_present(value_text, page_text):
+ if page_text is not None and _value_present_in_text(value_text, page_text):
pre_run_text = snapshot.pre_run_page_reference_text
if pre_run_text is None:
return None
- if not _boundary_delimited_present(value_text, _normalized_expected_text(pre_run_text)):
+ if not _value_present_in_text(value_text, _normalized_expected_text(pre_run_text)):
return CriterionVerdict(
criterion_id=criterion.id,
state="satisfied",
@@ -1305,6 +1307,51 @@ def _compact_expected_text(value: str) -> str:
return "".join(char for char in value if char.isalnum())
+def self_witnessed_numeric_substring(
+ criterion: CompletionCriterion,
+ snapshot: RunEvidenceSnapshot,
+ satisfied_source: EvidenceSourceKind | None,
+) -> bool:
+ """A registered number whose own evidence text carries it only inside a different quantity.
+
+ "8" reads as present in a tile's "-8.0%" delta while the visitor count it claims to be is 8.45K,
+ so the block's assertion becomes the only proof of itself (SKY-13332). A registered value that
+ stands on its own in the evidence, or that the evidence never mentions, is not this shape.
+ """
+ if satisfied_source != "registered_output_parameter":
+ return False
+ value_text = _carrier_scalar_text(criterion.expected_output_value)
+ if value_text is None or not value_text.replace(",", "").isdigit():
+ return False
+ for payload in snapshot.block_outputs.values():
+ if not isinstance(payload, Mapping):
+ continue
+ evidence_text = payload.get("evidence_text")
+ if not isinstance(evidence_text, str) or not evidence_text.strip():
+ continue
+ normalized = _normalized_expected_text(evidence_text)
+ if _value_present_in_text(value_text, normalized) and not _standalone_number_present(value_text, normalized):
+ return True
+ return False
+
+
+def _standalone_number_present(value_text: str, text: str) -> bool:
+ """Whether the value appears as a number in its own right, not inside a larger figure."""
+ needle = value_text.replace(",", "")
+ for match in re.finditer(r"\d[\d,]*(?:\.\d+)?", text):
+ token = match.group(0)
+ trailing = text[match.end() : match.end() + 1]
+ if trailing == "%" and token.replace(",", "") == needle:
+ continue
+ if token.replace(",", "") == needle:
+ return True
+ return False
+
+
+def _value_present_in_text(value_text: str, text: str) -> bool:
+ return _boundary_delimited_present(value_text, text)
+
+
def _normalize_output_path(path: str | None) -> str:
if not isinstance(path, str):
return ""
diff --git a/skyvern/forge/sdk/copilot/composition_browser_expressions.py b/skyvern/forge/sdk/copilot/composition_browser_expressions.py
index 3fcdc244a..de3ed3f9f 100644
--- a/skyvern/forge/sdk/copilot/composition_browser_expressions.py
+++ b/skyvern/forge/sdk/copilot/composition_browser_expressions.py
@@ -20,6 +20,7 @@ from skyvern.forge.sdk.copilot.composition_evidence import (
_MAX_RESULT_SAMPLE_ROWS,
_MAX_REVEAL_KEY_VALUE_RELATIONS,
_MAX_SELECT_OPTIONS,
+ _MAX_SELECTOR_CHARS,
_MAX_TABLE_HEADERS,
_MAX_VISIBLE_TEXT_EXCERPT_CHARS,
_MODAL_IDENTITY_PATTERNS,
@@ -368,6 +369,7 @@ _STRUCTURED_CONST_HEADER = (
f"const MAX_NAVIGATION_TARGETS={int(_MAX_NAVIGATION_TARGETS)};"
f"const MAX_RESULT_CONTAINERS={int(_MAX_RESULT_CONTAINERS)};"
f"const MAX_KEY_VALUE_RELATIONS={int(_MAX_KEY_VALUE_RELATIONS)};"
+ f"const MAX_SELECTOR_CHARS={int(_MAX_SELECTOR_CHARS)};"
f"const MAX_REVEAL_KEY_VALUE_RELATIONS={int(_MAX_REVEAL_KEY_VALUE_RELATIONS)};"
f"const MAX_TABLE_HEADERS={int(_MAX_TABLE_HEADERS)};"
f"const MAX_RESULT_SAMPLE_ROWS={int(_MAX_RESULT_SAMPLE_ROWS)};"
@@ -389,6 +391,19 @@ const FIELD_CAP = 2048;
const cap = (s) => (s.length > FIELD_CAP ? s.slice(0, FIELD_CAP) : s);
const attr = (el, k) => { const v = el && el.getAttribute ? el.getAttribute(k) : null; return typeof v === 'string' ? cap(v.trim()) : ''; };
const nodeText = (el) => { if (!el) return ''; return cap(String(el.textContent || '').replace(/\s+/g, ' ').trim()); };
+ const readsAsOneLeaf = (el) => {
+ if (!el || !el.children || !el.children.length) return true;
+ let bearing = 0;
+ const stack = Array.from(el.children);
+ while (stack.length) {
+ const cur = stack.pop();
+ const kids = Array.from(cur.children || []);
+ if (!kids.length) { if (nodeText(cur)) bearing += 1; }
+ else { for (let i = 0; i < kids.length; i++) stack.push(kids[i]); }
+ if (bearing > 1) return false;
+ }
+ return bearing <= 1;
+ };
const classesFor = (el) => Array.from((el && el.classList) || []).map((c) => String(c).trim()).filter(Boolean);
const cssAttr = (v) => String(v).split('\\').join('\\\\').split('"').join('\\"');
const simpleIdent = (v) => { if (!v) return false; if (!/[A-Za-z_-]/.test(v[0])) return false; for (let i = 1; i < v.length; i++) { if (!/[A-Za-z0-9_-]/.test(v[i])) return false; } return true; };
@@ -416,7 +431,13 @@ const structuralPath = (el) => {
if (pid && simpleIdent(pid)) { parts.unshift('#' + pid); break; }
node = parent;
}
- return parts.join(' > ');
+ const full = parts.join(' > ');
+ if (full.length <= MAX_SELECTOR_CHARS) return full;
+ for (let start = 1; start < parts.length; start++) {
+ const tail = parts.slice(start).join(' > ');
+ if (tail.length <= MAX_SELECTOR_CHARS && resolvesUniquely(tail, el)) return tail;
+ }
+ return full;
};
// The selector is a contract: the model clicks it and authors it into generated blocks, so an
// ambiguous or unresolvable guess costs a failed run rather than a retry. Every candidate is
@@ -683,11 +704,14 @@ const resultEntry = (node, tag) => {
}
return entry;
};
+ // Whole tokens, matching the parser twin: "arrow" contains "row", so a substring test hands every
+ // slot of the bounded budget to an icon sprite sheet before the document reaches a real result.
+ const matchesResultHint = (identity) => identity.split(/[^a-z0-9]+/).some((token) => !!token && RESULT_CONTAINER_HINTS.includes(token));
for (const node of all) {
const tag = (node.tagName || '').toLowerCase();
if (SKIP_TAGS.has(tag)) continue;
const identity = (attr(node, 'id') + ' ' + classesFor(node).join(' ')).toLowerCase();
- if (tag === 'table' || RESULT_CONTAINER_HINTS.some((h) => identity.includes(h))) {
+ if (tag === 'table' || matchesResultHint(identity)) {
if (resultContainers.length >= MAX_RESULT_CONTAINERS) { resultContainersTruncated = true; break; }
resultContainers.push(resultEntry(node, tag));
}
@@ -695,17 +719,137 @@ const resultEntry = (node, tag) => {
const keyValueRelations = [];
let keyValueRelationsTruncated = false;
+// Counting continues past the cap even though recording stops: whether a captured label is unique
+// on the page is what decides a bind, and stopping both turned "there is more" into "trust none".
+const foldedKeyCounts = new Map();
+const countFoldedKey = (text) => { const k = String(text || '').toLowerCase(); if (k) foldedKeyCounts.set(k, (foldedKeyCounts.get(k) || 0) + 1); };
+const walkedValueCounts = new Map();
+const countWalkedValue = (text) => { const v = String(text || ''); if (v) walkedValueCounts.set(v, (walkedValueCounts.get(v) || 0) + 1); };
const keyValueSkipTags = new Set(['body', 'form', 'html', 'table', 'tbody', 'thead', 'tr']);
+const nonContentChildTags = new Set(['style', 'script', 'noscript', 'svg', 'template']);
+const bareMagnitude = /^-?\$?\d{1,3}(?:[,\s]?\d{3})*(?:\.\d+)?[KMB]?$/i;
+const insidePageChrome = (node) => !!(node.closest && node.closest('nav,aside,header,footer,[role=navigation],[role=menu],[role=menubar],[role=banner],[role=complementary]'));
+const labelLike = (text) => text.length >= 2 && text.indexOf('{') < 0;
+const metricCardNodes = new Set();
+const insideMetricCard = (node) => { let ancestor = node; while (ancestor) { if (metricCardNodes.has(ancestor)) return true; ancestor = ancestor.parentElement; } return false; };
+// Mirror of _append_requested_target_relations: the labels the turn asked for are resolved before
+// any shape pass, so a tile that nests its figure deeper is still read by its own label rather than
+// skipped and replaced by whatever generic pair the page happens to offer.
+const isLeafEl = (el) => !el.children || el.children.length === 0;
+const withinEl = (el, ancestor) => { let c = el; while (c) { if (c === ancestor) return true; c = c.parentElement; } return false; };
+const valueBesideLabel = (labelEl) => {
+ let branch = labelEl;
+ let ancestor = labelEl.parentElement;
+ while (ancestor && ancestor.tagName !== 'BODY' && ancestor.tagName !== 'HTML') {
+ const candidates = [];
+ for (const leaf of Array.from(ancestor.querySelectorAll('*'))) {
+ if (!isLeafEl(leaf) || withinEl(leaf, branch)) continue;
+ if (bareMagnitude.test(nodeText(leaf))) candidates.push(leaf);
+ }
+ if (candidates.length > 1) return null;
+ if (candidates.length === 1) {
+ const valueLeaf = candidates[0];
+ const carrier = valueLeaf.parentElement;
+ if (!carrier) return null;
+ const kids = Array.from(carrier.children || []);
+ const childIndex = kids.indexOf(valueLeaf);
+ if (childIndex < 0) return null;
+ const labelText = nodeText(labelEl);
+ let labelSelector = '';
+ if (!kids.length || nodeText(kids[0]) !== labelText) {
+ labelSelector = selectorFor(labelEl);
+ if (!labelSelector || labelSelector.length > MAX_SELECTOR_CHARS || !resolvesUniquely(labelSelector, labelEl)) return null;
+ }
+ const sel = selectorFor(carrier);
+ if (!sel || sel.length > MAX_SELECTOR_CHARS) return null;
+ const matches = selectorMatchCount(sel);
+ if (!matches) return null;
+ let pos = -1;
+ try { pos = Array.from(document.querySelectorAll(sel)).indexOf(carrier); } catch (e) { pos = -1; }
+ if (pos < 0) return null;
+ return { owner: ancestor, relation: { key_text: labelText, label_selector: labelSelector, value_text: nodeText(valueLeaf), container_selector: sel, container_match_count: matches, container_position: pos, value_child_index: childIndex, direct_child_count: kids.length, visible: true, value_visible: true } };
+ }
+ branch = ancestor;
+ ancestor = ancestor.parentElement;
+ }
+ return null;
+};
+for (const target of REQUESTED_TARGETS) {
+ const wanted = String(target || '').trim().toLowerCase();
+ if (!wanted) continue;
+ for (const node of all) {
+ if (!isLeafEl(node) || !elementVisible(node) || insidePageChrome(node)) continue;
+ if (nodeText(node).trim().toLowerCase() !== wanted) continue;
+ const found = valueBesideLabel(node);
+ if (!found) continue;
+ countFoldedKey(found.relation.key_text);
+ countWalkedValue(found.relation.value_text);
+ keyValueRelations.push(found.relation);
+ metricCardNodes.add(found.owner);
+ break;
+ }
+}
for (const node of all) {
const tag = (node.tagName || '').toLowerCase();
- if (keyValueSkipTags.has(tag) || !elementVisible(node)) continue;
+ if (keyValueSkipTags.has(tag) || !elementVisible(node) || insidePageChrome(node) || insideMetricCard(node)) continue;
+ const children = Array.from(node.children || []);
+ if (children.length < 2 || children.length > 6) continue;
+ if (children.some((child) => nonContentChildTags.has((child.tagName || '').toLowerCase()))) continue;
+ const leaves = [];
+ let nestedOk = true;
+ for (let index = 0; index < children.length; index++) {
+ const child = children[index];
+ const grand = Array.from(child.children || []);
+ if (!grand.length) { const text = nodeText(child); if (text) leaves.push({ index: index, text: text, carrier: null, carrierIndex: 0, carrierCount: 0 }); continue; }
+ if (grand.some((g) => !readsAsOneLeaf(g))) { nestedOk = false; break; }
+ for (let gi = 0; gi < grand.length; gi++) { const text = nodeText(grand[gi]); if (text) leaves.push({ index: index, text: text, carrier: child, carrierIndex: gi, carrierCount: grand.length }); }
+ }
+ if (!nestedOk || !leaves.length || leaves.length > 8) continue;
+ const magnitudeLeaves2 = leaves.filter((leaf) => bareMagnitude.test(leaf.text));
+ if (magnitudeLeaves2.length !== 1) continue;
+ const cardValueIndex = magnitudeLeaves2[0].index;
+ const cardValueText = magnitudeLeaves2[0].text;
+ // The compiled read takes a direct child's whole text, so the relation is anchored wherever the
+ // figure is that child: the tile when the figure is a direct child, the row holding it when the
+ // figure sits beside a comparison.
+ let cardCarrier = node;
+ let cardChildIndex = cardValueIndex;
+ let cardChildCount = children.length;
+ if (nodeText(children[cardValueIndex]) !== cardValueText) {
+ const anchor = magnitudeLeaves2[0];
+ if (!anchor.carrier) continue;
+ cardCarrier = anchor.carrier;
+ cardChildIndex = anchor.carrierIndex;
+ cardChildCount = anchor.carrierCount;
+ }
+ const headingLeaves = leaves.filter((leaf) => leaf.index !== cardValueIndex && leaf.text.length <= 60 && labelLike(leaf.text) && !bareMagnitude.test(leaf.text));
+ const headingChildIndex = headingLeaves.length ? headingLeaves[0].index : 0;
+ if (!headingLeaves.length) continue;
+ const cardSelector = selectorFor(cardCarrier);
+ const cardMatches = selectorMatchCount(cardSelector);
+ if (!cardMatches) continue;
+ let cardPosition = -1;
+ try { cardPosition = Array.from(document.querySelectorAll(cardSelector)).indexOf(cardCarrier); } catch (e) { cardPosition = -1; }
+ if (cardPosition < 0) continue;
+ countFoldedKey(headingLeaves[0].text);
+ countWalkedValue(cardValueText);
+ if (keyValueRelations.length >= MAX_KEY_VALUE_RELATIONS) { keyValueRelationsTruncated = true; continue; }
+ keyValueRelations.push({ key_text: headingLeaves[0].text, value_text: cardValueText, container_selector: cardSelector, container_match_count: cardMatches, container_position: cardPosition, value_child_index: cardChildIndex, label_child_index: (cardCarrier === node ? headingChildIndex : -1), direct_child_count: cardChildCount, visible: true, value_visible: true });
+ metricCardNodes.add(node);
+}
+for (const node of all) {
+ const tag = (node.tagName || '').toLowerCase();
+ if (keyValueSkipTags.has(tag) || !elementVisible(node) || insidePageChrome(node) || insideMetricCard(node)) continue;
const children = Array.from(node.children || []);
if (children.length !== 2) continue;
if (children[0].children && children[0].children.length > 0) continue;
+ if (children.some((child) => nonContentChildTags.has((child.tagName || '').toLowerCase()))) continue;
const keyText = nodeText(children[0]);
const valueText = nodeText(children[1]);
- if (!keyText || keyText.length > 120 || !valueText || keyText === valueText) continue;
- if (keyValueRelations.length >= MAX_KEY_VALUE_RELATIONS) { keyValueRelationsTruncated = true; break; }
+ if (!keyText || keyText.length > 120 || !valueText || keyText === valueText || !labelLike(keyText)) continue;
+ countFoldedKey(keyText);
+ countWalkedValue(valueText);
+ if (keyValueRelations.length >= MAX_KEY_VALUE_RELATIONS) { keyValueRelationsTruncated = true; continue; }
const selector = selectorFor(node);
const matches = selectorMatchCount(selector);
if (!matches) continue;
@@ -722,7 +866,7 @@ let revealRelationCount = 0;
let revealRelationsTruncated = false;
for (const node of all) {
const tag = (node.tagName || '').toLowerCase();
- if (keyValueSkipTags.has(tag) || !elementVisible(node)) continue;
+ if (keyValueSkipTags.has(tag) || !elementVisible(node) || insidePageChrome(node) || insideMetricCard(node)) continue;
if (!matchesResultHintToken(node)) continue;
const children = Array.from(node.children || []);
if (children.length < 3 || children.length > 6) continue;
@@ -745,11 +889,12 @@ for (const node of all) {
if (!valueText || keyText === valueText) continue;
valueLeaves.push({ index: i, valueText: valueText });
}
- const revealKeyText = valueLeaves.length === 1 ? keyText : '';
+ const magnitudeLeaves = valueLeaves.filter((leaf) => bareMagnitude.test(leaf.valueText));
+ const designatedIndex = valueLeaves.length === 1 ? valueLeaves[0].index : (magnitudeLeaves.length === 1 ? magnitudeLeaves[0].index : null);
let capped = false;
for (const leaf of valueLeaves) {
if (keyValueRelations.length >= MAX_KEY_VALUE_RELATIONS || revealRelationCount >= MAX_REVEAL_KEY_VALUE_RELATIONS) { revealRelationsTruncated = true; capped = true; break; }
- keyValueRelations.push({ key_text: revealKeyText, value_text: leaf.valueText, container_selector: selector, container_match_count: matches, container_position: position, value_child_index: leaf.index, direct_child_count: children.length, visible: true, value_visible: true });
+ keyValueRelations.push({ key_text: leaf.index === designatedIndex ? keyText : '', value_text: leaf.valueText, container_selector: selector, container_match_count: matches, container_position: position, value_child_index: leaf.index, direct_child_count: children.length, visible: true, value_visible: true });
revealRelationCount++;
}
if (capped) break;
@@ -848,7 +993,9 @@ return JSON.stringify({
navigation_targets: navTargets,
result_containers: resultContainers,
result_containers_truncated: resultContainersTruncated,
- key_value_relations: keyValueRelations,
+ key_value_relations: keyValueRelations.map((r) => Object.assign({}, r,
+ String(r.key_text || '') ? { key_text_walked_count: foldedKeyCounts.get(String(r.key_text).toLowerCase()) || 1 } : {},
+ String(r.value_text || '') ? { value_text_walked_count: walkedValueCounts.get(String(r.value_text)) || 1 } : {})),
key_value_relations_truncated: keyValueRelationsTruncated,
reveal_relations_truncated: revealRelationsTruncated,
clickable_controls: clickableControls,
@@ -861,4 +1008,16 @@ return JSON.stringify({
});
"""
-COMPOSITION_STRUCTURED_EVIDENCE_EXPRESSION = "(() => {" + _STRUCTURED_CONST_HEADER + _STRUCTURED_EVIDENCE_BODY + "})()"
+
+def composition_structured_evidence_expression(requested_targets: tuple[str, ...] = ()) -> str:
+ """The capture expression, told which labels the turn asked for.
+
+ Capture that does not know the request can only guess which of a page's relations matters, and a
+ tile whose shape it fails to recognise is replaced by whatever generic pair sits nearby.
+ """
+ targets = [target.strip() for target in requested_targets if isinstance(target, str) and target.strip()]
+ header = f"const REQUESTED_TARGETS={json.dumps(targets[:_MAX_KEY_VALUE_RELATIONS])};"
+ return "(() => {" + header + _STRUCTURED_CONST_HEADER + _STRUCTURED_EVIDENCE_BODY + "})()"
+
+
+COMPOSITION_STRUCTURED_EVIDENCE_EXPRESSION = composition_structured_evidence_expression()
diff --git a/skyvern/forge/sdk/copilot/composition_evidence.py b/skyvern/forge/sdk/copilot/composition_evidence.py
index f71084388..6d427d78f 100644
--- a/skyvern/forge/sdk/copilot/composition_evidence.py
+++ b/skyvern/forge/sdk/copilot/composition_evidence.py
@@ -55,6 +55,8 @@ _RESULT_CONTAINER_HINTS: frozenset[str] = frozenset({"result", "results", "recor
_MAX_FORMS = 5
_MAX_FIELDS_PER_FORM = 20
_MAX_RESULT_CONTAINERS = 8
+# The cap _schema_text applies to a relation's value; at it, the text is a prefix, not the value.
+_MAX_RELATION_VALUE_CHARS = 240
_MAX_KEY_VALUE_RELATIONS = 24
_MAX_REVEAL_KEY_VALUE_RELATIONS = 8
_MAX_TABLE_HEADERS = 12
@@ -1352,7 +1354,17 @@ def _structural_path(node: Any) -> str:
parts.insert(0, f"#{parent_id}")
break
current = parent
- return " > ".join(parts)
+ full = " > ".join(parts)
+ if len(full) <= _MAX_SELECTOR_CHARS:
+ return full
+ # A tile nested deeply in id-less markup outgrows the budget, and a path cut to fit ends
+ # mid-token. The shortest tail that still resolves to this node alone says the same thing in
+ # fewer characters, and the caller verifies it before handing it out either way.
+ for start in range(1, len(parts)):
+ tail = " > ".join(parts[start:])
+ if len(tail) <= _MAX_SELECTOR_CHARS and _resolves_uniquely(node, tail):
+ return tail
+ return full
def _control_label(node: Any) -> str:
@@ -1372,6 +1384,15 @@ def _control_label(node: Any) -> str:
return ""
+_MAX_SELECTOR_CHARS = 160
+
+
+def _bounded_selector(selector: str) -> str:
+ """A selector short enough to survive intact, or nothing. Cutting one to a length budget can end
+ it mid-token (``div:nth-of-ty``), and the generated block then raises on querySelectorAll."""
+ return selector if len(selector) <= _MAX_SELECTOR_CHARS else ""
+
+
def _selector_for(node: Any) -> str:
"""Mirror of ``selectorFor`` in composition_browser_expressions. The selector is a contract: the
model clicks it and authors it into generated blocks, so every candidate is verified to match
@@ -1479,7 +1500,7 @@ def _clickable_controls_html(soup: Any, *, used_selectors: set[str]) -> list[dic
text = _schema_text(_clickable_control_text(node), 120)
selector = _clickable_control_selector(node)
if selector and selector not in seen_selectors and _selector_is_live_unique_in_soup(soup, selector):
- controls.append({"text": text, "selector": selector[:160], "tag": tag_name})
+ controls.append({"text": text, "selector": _bounded_selector(selector), "tag": tag_name})
seen_selectors.add(selector)
if text:
seen_text.add(text)
@@ -1577,26 +1598,314 @@ def _selector_match_count(soup: Any, selector: str) -> int:
return 0
-def _key_value_relations(soup: Any) -> tuple[list[dict[str, Any]], bool, bool]:
- relations: list[dict[str, Any]] = []
+_PAGE_CHROME_TAGS = frozenset({"nav", "aside", "header", "footer"})
+_PAGE_CHROME_ROLES = frozenset({"navigation", "menu", "menubar", "banner", "complementary"})
+_NON_CONTENT_CHILD_TAGS = frozenset({"style", "script", "noscript", "svg", "template"})
+
+
+def _inside_page_chrome(node: Any) -> bool:
+ """Site navigation and chrome repeat page vocabulary (a sidebar "Visitors" item shadows the
+ metric tile) and flood the relation cap with junk pairs, so no relation binds inside them."""
+ current = node
+ while current is not None and getattr(current, "name", None):
+ if str(current.name).lower() in _PAGE_CHROME_TAGS:
+ return True
+ role = str(current.get("role") or "").lower() if hasattr(current, "get") else ""
+ if role in _PAGE_CHROME_ROLES:
+ return True
+ current = current.parent
+ return False
+
+
+def _label_like(key_text: str) -> bool:
+ return len(key_text) >= 2 and "{" not in key_text
+
+
+def _reads_as_one_leaf(node: Any) -> bool:
+ """Whether a node's text comes from a single text-bearing descendant.
+
+ A tile's figure sits beside a delta rendered as an arrow plus its text, so treating any nested
+ grandchild as structure abandoned the whole tile and left the delta to the positional builder —
+ the figure was present the entire time (SKY-13226). Counts text-bearing descendants rather than
+ naming decorative tags, so an icon of any element type stays decoration.
+ """
+ if node.find(True) is None:
+ return True
+ bearing = [
+ descendant
+ for descendant in node.find_all(True)
+ if _schema_text(_node_text(descendant), 240) and descendant.find(True) is None
+ ]
+ return len(bearing) <= 1
+
+
+def _append_metric_card_relations(
+ soup: Any,
+ relations: list[dict[str, Any]],
+ folded_key_counts: dict[str, int],
+ walked_value_counts: dict[str, int],
+ owned_by_target: set[int] | None = None,
+) -> tuple[list[Any], bool]:
+ """Emit label→value relations for metric-card tiles (a heading beside exactly one bare-magnitude
+ figure, deltas and comparisons around it), which fit neither the 2-child pair nor the reveal
+ shape and so left the requested value structurally invisible (SKY-13226).
+
+ Reports whether the cap dropped a card: a channel that lost relations without saying so reads as
+ a complete list, which is the one thing a binder is allowed to trust.
+ """
+ cards: list[Any] = []
+ # A tile the targeted pass already resolved is not re-emitted here; both passes reach the same
+ # figure, and two relations for one tile read as two observations of it.
+ card_ids: set[int] = set(owned_by_target or ())
truncated = False
for node in soup.find_all(True):
tag_name = str(getattr(node, "name", "") or "").lower()
if tag_name in {"body", "form", "html", "table", "tbody", "thead", "tr"}:
continue
+ if _inside_page_chrome(node) or id(node) in card_ids or any(id(parent) in card_ids for parent in node.parents):
+ continue
+ children = [child for child in node.find_all(recursive=False) if child.name]
+ if not (2 <= len(children) <= 6):
+ continue
+ if any(str(child.name).lower() in _NON_CONTENT_CHILD_TAGS for child in children):
+ continue
+ leaves: list[tuple[int, str]] = []
+ # Where each leaf sits when it is a grandchild: the child holding it, and its index within.
+ leaf_anchors: dict[tuple[int, str], tuple[Any, int, int]] = {}
+ nested_ok = True
+ for index, child in enumerate(children):
+ grandchildren = [grand for grand in child.find_all(recursive=False) if grand.name]
+ if not grandchildren:
+ text = _schema_text(_node_text(child), 240)
+ if text:
+ leaves.append((index, text))
+ continue
+ if any(not _reads_as_one_leaf(grand) for grand in grandchildren):
+ nested_ok = False
+ break
+ for grand_index, grand in enumerate(grandchildren):
+ text = _schema_text(_node_text(grand), 240)
+ if text:
+ leaves.append((index, text))
+ leaf_anchors.setdefault((index, text), (child, grand_index, len(grandchildren)))
+ if not nested_ok or not leaves or len(leaves) > 8:
+ continue
+ magnitudes = [(index, text) for index, text in leaves if _BARE_MAGNITUDE_RE.fullmatch(text)]
+ if len(magnitudes) != 1:
+ continue
+ value_index, value_text = magnitudes[0]
+ # The compiled read takes a direct child's whole text, so the relation is anchored wherever
+ # the figure is that child: the tile when the figure is a direct child, and the row holding
+ # it when the figure sits beside a comparison. Anchoring only at the tile left a figure that
+ # shares its row with "vs. N prior" unreadable, and the heading fell to the positional
+ # builder beside the delta.
+ value_carrier, value_child_index, value_child_count = node, value_index, len(children)
+ if _schema_text(_node_text(children[value_index]), 240) != value_text:
+ anchor = leaf_anchors.get((value_index, value_text))
+ if anchor is None:
+ continue
+ value_carrier, value_child_index, value_child_count = anchor
+ heading_leaves = [
+ (index, text)
+ for index, text in leaves
+ if index != value_index and len(text) <= 60 and _label_like(text) and not _BARE_MAGNITUDE_RE.fullmatch(text)
+ ]
+ if not heading_leaves:
+ continue
+ heading_index, headings = heading_leaves[0][0], [text for _index, text in heading_leaves]
+ selector = _bounded_selector(_selector_for(value_carrier))
+ match_count = _selector_match_count(soup, selector)
+ if match_count <= 0:
+ continue
+ try:
+ position = soup.select(selector).index(value_carrier)
+ except ValueError:
+ continue
+ folded_heading = headings[0][:120].lower()
+ folded_key_counts[folded_heading] = folded_key_counts.get(folded_heading, 0) + 1
+ # Counted before the cap, like the heading beside it: a value the cap drops still appeared on
+ # the page, and a witness reading the count as page-wide uniqueness would bind the wrong tile.
+ if value_text:
+ walked_value_counts[value_text] = walked_value_counts.get(value_text, 0) + 1
+ if len(relations) >= _MAX_KEY_VALUE_RELATIONS:
+ truncated = True
+ continue
+ relations.append(
+ {
+ "key_text": headings[0][:120],
+ "value_text": value_text,
+ "container_selector": selector,
+ "container_match_count": match_count,
+ "container_position": position,
+ "value_child_index": value_child_index,
+ # A tile that prints its figure before its heading ("1.22K logs found") puts the label
+ # somewhere other than the first child, and a read proving the label at child zero
+ # then fails on a page that plainly shows both. Re-anchoring onto the figure's own row
+ # leaves the heading outside it entirely, and then no child index names the label.
+ "label_child_index": heading_index if value_carrier is node else -1,
+ "direct_child_count": value_child_count,
+ "visible": True,
+ "value_visible": True,
+ }
+ )
+ cards.append(node)
+ card_ids.add(id(node))
+ return cards, truncated
+
+
+def _is_leaf_element(node: Any) -> bool:
+ return bool(getattr(node, "name", None)) and node.find(True) is None
+
+
+def _within(node: Any, ancestor: Any) -> bool:
+ current = node
+ while current is not None:
+ if current is ancestor:
+ return True
+ current = getattr(current, "parent", None)
+ return False
+
+
+def _value_beside_requested_label(soup: Any, label_node: Any) -> tuple[dict[str, Any], Any] | None:
+ """The value a labelled tile carries, found by widening from the label until one candidate remains.
+
+ Anchoring on the requested label makes this an identity search rather than a guess at which
+ containers look like metric tiles, so a figure the tile nests deeper is still its value. Widening
+ stops at the first level that offers exactly one candidate and abstains where several do: markup
+ that cannot say which number the label owns is not evidence that any of them is.
+ """
+ branch = label_node
+ ancestor = getattr(label_node, "parent", None)
+ while ancestor is not None and str(getattr(ancestor, "name", "") or "") not in {"", "body", "html", "[document]"}:
+ candidates = [
+ leaf
+ for leaf in ancestor.find_all(True)
+ if _is_leaf_element(leaf)
+ and not _within(leaf, branch)
+ and _BARE_MAGNITUDE_RE.fullmatch(_schema_text(_node_text(leaf), 240))
+ ]
+ if len(candidates) > 1:
+ return None
+ if len(candidates) == 1:
+ value_leaf = candidates[0]
+ carrier = value_leaf.parent
+ children = [child for child in carrier.find_all(recursive=False) if child.name]
+ if value_leaf not in children:
+ return None
+ # A generated scalar read proves identity by re-reading the label. Where the label is not
+ # the value's sibling it is proven at its own anchor instead, and a label with neither is
+ # unreadable: emitting it anyway builds a block that raises on its own guard.
+ label_text = _schema_text(_node_text(label_node), 120)
+ label_selector = ""
+ if not children or _schema_text(_node_text(children[0]), 120) != label_text:
+ label_selector = _bounded_selector(_selector_for(label_node))
+ if not label_selector or not _resolves_uniquely(label_node, label_selector):
+ return None
+ selector = _bounded_selector(_selector_for(carrier))
+ match_count = _selector_match_count(soup, selector) if selector else 0
+ if not selector or match_count <= 0:
+ return None
+ try:
+ position = soup.select(selector).index(carrier)
+ except ValueError:
+ return None
+ return (
+ {
+ "key_text": label_text,
+ "label_selector": label_selector,
+ "value_text": _schema_text(_node_text(value_leaf), 240),
+ "container_selector": selector,
+ "container_match_count": match_count,
+ "container_position": position,
+ "value_child_index": children.index(value_leaf),
+ "direct_child_count": len(children),
+ "visible": True,
+ "value_visible": True,
+ },
+ ancestor,
+ )
+ branch = ancestor
+ ancestor = getattr(ancestor, "parent", None)
+ return None
+
+
+def _append_requested_target_relations(
+ soup: Any, relations: list[dict[str, Any]], folded_key_counts: dict[str, int], requested_targets: tuple[str, ...]
+) -> set[int]:
+ owned: set[int] = set()
+ for target in requested_targets:
+ folded = target.strip().casefold()
+ if not folded:
+ continue
+ for node in soup.find_all(True):
+ if not _is_leaf_element(node) or _inside_page_chrome(node):
+ continue
+ if _node_text(node).strip().casefold() != folded:
+ continue
+ found = _value_beside_requested_label(soup, node)
+ if found is None:
+ continue
+ relation, carrier = found
+ key = str(relation.get("key_text") or "").lower()
+ if key:
+ folded_key_counts[key] = folded_key_counts.get(key, 0) + 1
+ relations.append(relation)
+ owned.add(id(carrier))
+ break
+ return owned
+
+
+def _key_value_relations(soup: Any, requested_targets: tuple[str, ...] = ()) -> tuple[list[dict[str, Any]], bool, bool]:
+ relations: list[dict[str, Any]] = []
+ # How often each folded label occurs across the whole page, counted past the cap so a bind can be
+ # decided from a truncated capture instead of refused wholesale.
+ folded_key_counts: dict[str, int] = {}
+ # Values are counted alongside labels and for the same reason: a witness binds on the value,
+ # so whether the page shows that value once is what decides the bind on a truncated channel.
+ # Counted as displayed rather than folded, because a witness compares displayed text.
+ walked_value_counts: dict[str, int] = {}
+ # The labels the turn asked for are resolved before anything else, so the cap cannot starve them
+ # and the shape passes below cannot claim their tiles first.
+ owned_by_target = _append_requested_target_relations(soup, relations, folded_key_counts, requested_targets)
+ for relation in relations:
+ early_value = str(relation.get("value_text") or "")
+ if early_value:
+ walked_value_counts[early_value] = walked_value_counts.get(early_value, 0) + 1
+ cards, truncated = _append_metric_card_relations(
+ soup, relations, folded_key_counts, walked_value_counts, owned_by_target
+ )
+ card_ids = {id(card) for card in cards} | owned_by_target
+ for node in soup.find_all(True):
+ tag_name = str(getattr(node, "name", "") or "").lower()
+ if tag_name in {"body", "form", "html", "table", "tbody", "thead", "tr"}:
+ continue
+ if _inside_page_chrome(node):
+ continue
+ if id(node) in card_ids or any(id(parent) in card_ids for parent in node.parents):
+ continue
children = [child for child in node.find_all(recursive=False) if child.name]
if len(children) != 2:
continue
if children[0].find(True) is not None:
continue
+ if any(str(child.name).lower() in _NON_CONTENT_CHILD_TAGS for child in children):
+ continue
key_text = _schema_text(_node_text(children[0]), 120)
value_text = _schema_text(_node_text(children[1]), 240)
- if not key_text or not value_text or key_text == value_text:
- continue
+ if not key_text or not value_text or key_text == value_text or not _label_like(key_text):
+ continue
+ # Counting continues past the cap even though recording stops: whether a captured label is
+ # unique on the page is what decides a bind, and stopping the count as well as the payload
+ # turned "there is more beyond this" into "nothing here can be trusted".
+ # Simple lowercase rather than casefold, because the page-side twin counts with
+ # String.toLowerCase and a count only means something if both producers merge alike.
+ folded = key_text.lower()
+ folded_key_counts[folded] = folded_key_counts.get(folded, 0) + 1
+ walked_value_counts[value_text] = walked_value_counts.get(value_text, 0) + 1
if len(relations) >= _MAX_KEY_VALUE_RELATIONS:
truncated = True
- break
- selector = _selector_for(node)[:160]
+ continue
+ selector = _bounded_selector(_selector_for(node))
match_count = _selector_match_count(soup, selector)
if match_count <= 0:
continue
@@ -1618,23 +1927,85 @@ def _key_value_relations(soup: Any) -> tuple[list[dict[str, Any]], bool, bool]:
"value_visible": True,
}
)
- reveal_truncated = _append_reveal_shape_relations(soup, relations)
+ reveal_truncated = _append_reveal_shape_relations(soup, relations, card_ids)
+ for relation in relations:
+ folded = str(relation.get("key_text") or "").lower()
+ if folded:
+ relation["key_text_walked_count"] = folded_key_counts.get(folded, 1)
+ value_text = str(relation.get("value_text") or "")
+ if value_text:
+ relation["value_text_walked_count"] = walked_value_counts.get(value_text, 1)
+ if len(value_text) >= _MAX_RELATION_VALUE_CHARS:
+ relation["value_truncated"] = True
return relations, truncated, reveal_truncated
+def clearable_dismiss_texts(evidence: dict[str, Any]) -> set[str]:
+ """The texts of the dismiss controls the captured dialogs offer."""
+ texts: set[str] = set()
+ for overlay in evidence.get("modal_overlays") or []:
+ if not isinstance(overlay, dict):
+ continue
+ for control in overlay.get("dismiss_controls") or []:
+ text = str((control or {}).get("text") or "").strip() if isinstance(control, dict) else str(control).strip()
+ if text:
+ texts.add(text)
+ return texts
+
+
+def packet_describes_a_clearable_overlay(evidence: dict[str, Any]) -> bool:
+ """Whether the relations captured are the overlay's own controls rather than the page's.
+
+ Capture reads what is in front, so an overlay can leave the packet describing the dialog instead
+ of the page behind it: a log query whose count was plainly on screen came back carrying only the
+ dialog's two buttons. Composing an extraction from that packet spends the round on evidence the
+ page never offered, while the control that clears it was collected and then dropped. Judged by
+ whether the dialog's own controls account for every relation, so a dialog beside readable page
+ content still composes.
+ """
+ dismiss_texts = clearable_dismiss_texts(evidence)
+ if not dismiss_texts:
+ return False
+ relations = [relation for relation in evidence.get("key_value_relations") or [] if isinstance(relation, dict)]
+ if not relations:
+ return False
+ return all(
+ str(relation.get("key_text") or "").strip() in dismiss_texts
+ or str(relation.get("value_text") or "").strip() in dismiss_texts
+ for relation in relations
+ )
+
+
def _matches_result_hint_token(node: Any) -> bool:
node_id = str(node.get("id") or "") if hasattr(node, "get") else ""
raw = f"{node_id} {' '.join(_classes_for(node))}".lower()
return any(token in _RESULT_CONTAINER_HINTS for token in re.split(r"[^a-z0-9]+", raw) if token)
-def _append_reveal_shape_relations(soup: Any, relations: list[dict[str, Any]]) -> bool:
+_BARE_MAGNITUDE_RE = re.compile(r"-?\$?\d{1,3}(?:[,\s]?\d{3})*(?:\.\d+)?[KMB]?", re.IGNORECASE)
+
+
+def _designated_metric_leaf_index(value_leaves: list[tuple[int, str]]) -> int | None:
+ """The one leaf of a multi-leaf tile whose heading survives, or None when nothing distinguishes
+ them. Bare magnitude is a heuristic: a tile's value sits at no fixed child index."""
+
+ if len(value_leaves) == 1:
+ return value_leaves[0][0]
+ magnitudes = [index for index, value_text in value_leaves if _BARE_MAGNITUDE_RE.fullmatch(value_text)]
+ return magnitudes[0] if len(magnitudes) == 1 else None
+
+
+def _append_reveal_shape_relations(soup: Any, relations: list[dict[str, Any]], card_ids: set[int]) -> bool:
reveal_count = 0
reveal_truncated = False
for node in soup.find_all(True):
tag_name = str(getattr(node, "name", "") or "").lower()
if tag_name in {"body", "form", "html", "table", "tbody", "thead", "tr"}:
continue
+ if _inside_page_chrome(node):
+ continue
+ if id(node) in card_ids or any(id(parent) in card_ids for parent in node.parents):
+ continue
if not _matches_result_hint_token(node):
continue
children = [child for child in node.find_all(recursive=False) if child.name]
@@ -1648,7 +2019,7 @@ def _append_reveal_shape_relations(soup: Any, relations: list[dict[str, Any]]) -
key_text = _schema_text(_node_text(heading), 240)
if not key_text or len(key_text) > 120:
continue
- selector = _selector_for(node)[:160]
+ selector = _bounded_selector(_selector_for(node))
match_count = _selector_match_count(soup, selector)
if match_count <= 0:
continue
@@ -1662,7 +2033,7 @@ def _append_reveal_shape_relations(soup: Any, relations: list[dict[str, Any]]) -
if not value_text or key_text == value_text:
continue
value_leaves.append((index, value_text))
- reveal_key_text = key_text if len(value_leaves) == 1 else ""
+ designated_index = _designated_metric_leaf_index(value_leaves)
capped = False
for index, value_text in value_leaves:
if len(relations) >= _MAX_KEY_VALUE_RELATIONS or reveal_count >= _MAX_REVEAL_KEY_VALUE_RELATIONS:
@@ -1671,7 +2042,7 @@ def _append_reveal_shape_relations(soup: Any, relations: list[dict[str, Any]]) -
break
relations.append(
{
- "key_text": reveal_key_text,
+ "key_text": key_text if index == designated_index else "",
"value_text": value_text,
"container_selector": selector,
"container_match_count": match_count,
@@ -1691,7 +2062,7 @@ def _append_reveal_shape_relations(soup: Any, relations: list[dict[str, Any]]) -
def _result_container_entry(node: Any, *, soup: Any) -> dict[str, Any]:
tag_name = str(node.name or "").lower()
node_id = str(node.get("id") or "")
- selector = _selector_for(node)[:160]
+ selector = _bounded_selector(_selector_for(node))
entry: dict[str, Any] = {
"tag": tag_name,
"id": node_id[:120],
@@ -1759,7 +2130,7 @@ def _challenge_control_entry(node: Any) -> dict[str, Any]:
"name": _attr_value(node, "name")[:120],
"class": " ".join(_classes_for(node)[:5])[:160],
"type": control_type,
- "selector": _selector_for(node)[:160],
+ "selector": _bounded_selector(_selector_for(node)),
"text": _schema_text(
_node_text(node) or _attr_value(node, "value") or _attr_value(node, "aria-label"),
200,
@@ -1819,8 +2190,10 @@ def _challenge_controls(soup: Any) -> list[dict[str, Any]]:
# trigger the visual fallback but must not read as a rendered control.
if _is_hidden_modal_candidate(node):
continue
- selector = _selector_for(node)[:160]
- if selector in seen_selectors:
+ selector = _bounded_selector(_selector_for(node))
+ # A selector too long to survive comes back empty, and every such node would otherwise
+ # dedupe against the first one.
+ if selector and selector in seen_selectors:
continue
seen_selectors.add(selector)
controls.append(_challenge_control_entry(node))
@@ -1880,8 +2253,10 @@ def _modal_dismiss_controls(node: Any) -> list[dict[str, Any]]:
for control in node.find_all(["button", "a", "input"]):
if len(controls) >= _MAX_MODAL_DISMISS_CONTROLS:
break
- selector = _selector_for(control)[:160]
- if selector in seen_selectors:
+ selector = _bounded_selector(_selector_for(control))
+ # A selector too long to survive comes back empty, and every such node would otherwise
+ # dedupe against the first one.
+ if selector and selector in seen_selectors:
continue
# Every control the dialog offers is reported. A keyword list cannot name every way a dialog
# closes ("No, keep ...", an icon-only glyph), and filtering on one leaves the agent looking
@@ -1909,7 +2284,7 @@ def _modal_overlay_entry(node: Any) -> dict[str, Any]:
"aria_modal": _attr_value(node, "aria-modal").strip().lower() == "true",
"id": _attr_value(node, "id")[:120],
"class": " ".join(_classes_for(node)[:5])[:160],
- "selector": _selector_for(node)[:160],
+ "selector": _bounded_selector(_selector_for(node)),
"text": _schema_text(_node_text(node), 240),
"dismiss_controls": _modal_dismiss_controls(node),
}
@@ -1925,8 +2300,10 @@ def _modal_overlays(nodes: Iterable[Any]) -> list[dict[str, Any]]:
continue
if _is_hidden_modal_candidate(node):
continue
- selector = _selector_for(node)[:160]
- if selector in seen_selectors:
+ selector = _bounded_selector(_selector_for(node))
+ # A selector too long to survive comes back empty, and every such node would otherwise
+ # dedupe against the first one.
+ if selector and selector in seen_selectors:
continue
entry = _modal_overlay_entry(node)
dismiss_controls = entry.get("dismiss_controls")
@@ -1975,7 +2352,9 @@ def _anti_bot_indicators(html: str, page_title: str) -> list[str]:
return [pattern for pattern in _ANTI_BOT_PATTERNS if pattern in haystack]
-def parse_composition_html(html: str, *, inspected_url: str, current_url: str) -> dict[str, Any]:
+def parse_composition_html(
+ html: str, *, inspected_url: str, current_url: str, requested_targets: tuple[str, ...] = ()
+) -> dict[str, Any]:
"""Extract a compact page schema for build-time workflow composition."""
if BeautifulSoup is None:
@@ -2026,7 +2405,7 @@ def parse_composition_html(html: str, *, inspected_url: str, current_url: str) -
"class": " ".join(_classes_for(node)[:5])[:160],
"type": field_type[:40],
"disabled": _control_disabled(node),
- "selector": _selector_for(node)[:160],
+ "selector": _bounded_selector(_selector_for(node)),
}
)
continue
@@ -2049,7 +2428,7 @@ def parse_composition_html(html: str, *, inspected_url: str, current_url: str) -
"disabled": _control_disabled(node),
"checked": bool(node.has_attr("checked")),
"options": _select_options(node) if tag_name == "select" else [],
- "selector": _selector_for(node)[:160],
+ "selector": _bounded_selector(_selector_for(node)),
}
)
forms.append(
@@ -2077,7 +2456,7 @@ def parse_composition_html(html: str, *, inspected_url: str, current_url: str) -
nav_entry: dict[str, Any] = {
"text": _schema_text(text, 160),
"href": resolved_href[:300],
- "selector": _selector_for(link)[:160],
+ "selector": _bounded_selector(_selector_for(link)),
}
download_kind = classify_download_affordance(
href=resolved_href,
@@ -2101,7 +2480,9 @@ def parse_composition_html(html: str, *, inspected_url: str, current_url: str) -
break
result_containers.append(_result_container_entry(node, soup=soup))
- key_value_relations, key_value_relations_truncated, reveal_relations_truncated = _key_value_relations(soup)
+ key_value_relations, key_value_relations_truncated, reveal_relations_truncated = _key_value_relations(
+ soup, requested_targets
+ )
reveal_relations_truncated = reveal_relations_truncated and not key_value_relations_truncated
used_selectors: set[str] = set()
@@ -2211,7 +2592,7 @@ def _structured_form(form: Any) -> dict[str, Any] | None:
"disabled": node.get("disabled") is True,
"checked": node.get("checked") is True,
"options": _structured_select_options(node.get("options")),
- "selector": _structured_str(node.get("selector"))[:160],
+ "selector": _bounded_selector(_structured_str(node.get("selector"))),
}
)
submit_controls: list[dict[str, Any]] = []
@@ -2227,7 +2608,7 @@ def _structured_form(form: Any) -> dict[str, Any] | None:
"class": _structured_classes(control.get("class")),
"type": (_structured_str(control.get("type")) or "").lower()[:40],
"disabled": control.get("disabled") is True,
- "selector": _structured_str(control.get("selector"))[:160],
+ "selector": _bounded_selector(_structured_str(control.get("selector"))),
}
)
return {
@@ -2257,7 +2638,7 @@ def _structured_navigation_targets(value: Any, *, base_url: str) -> list[dict[st
entry: dict[str, Any] = {
"text": _schema_text(_structured_str(link.get("text")), 160),
"href": href[:300],
- "selector": _structured_str(link.get("selector"))[:160],
+ "selector": _bounded_selector(_structured_str(link.get("selector"))),
}
download_kind = classify_download_affordance(
href=href,
@@ -2279,7 +2660,7 @@ def _structured_result_containers(value: Any) -> list[dict[str, Any]]:
if not isinstance(node, dict):
continue
tag_name = (_structured_str(node.get("tag")) or "").lower()
- selector = _structured_str(node.get("selector"))[:160]
+ selector = _bounded_selector(_structured_str(node.get("selector")))
entry: dict[str, Any] = {
"tag": tag_name,
"id": _structured_str(node.get("id"))[:120],
@@ -2381,7 +2762,7 @@ def _structured_key_value_relations(value: Any) -> list[dict[str, Any]]:
if not isinstance(item, dict):
continue
key_text = _schema_text(_structured_str(item.get("key_text")), 120)
- selector = _structured_str(item.get("container_selector"))[:160]
+ selector = _bounded_selector(_structured_str(item.get("container_selector")))
match_count = item.get("container_match_count")
position = item.get("container_position")
child_index = item.get("value_child_index")
@@ -2406,19 +2787,34 @@ def _structured_key_value_relations(value: Any) -> list[dict[str, Any]]:
continue
if match_count <= position:
continue
- relations.append(
- {
- "key_text": key_text,
- "value_text": value_text,
- "container_selector": selector,
- "container_match_count": match_count,
- "container_position": position,
- "value_child_index": child_index,
- "direct_child_count": child_count,
- "visible": item.get("visible") is True,
- "value_visible": item.get("value_visible") is True,
- }
- )
+ relation = {
+ "key_text": key_text,
+ "value_text": value_text,
+ "container_selector": selector,
+ "container_match_count": match_count,
+ "container_position": position,
+ "value_child_index": child_index,
+ "direct_child_count": child_count,
+ "visible": item.get("visible") is True,
+ "value_visible": item.get("value_visible") is True,
+ }
+ label_child_index = item.get("label_child_index")
+ # -1 is meaningful: the page-side capture emits it for a tile whose heading sits outside the
+ # value's row, and dropping it reads downstream as "the label is child zero".
+ if isinstance(label_child_index, int) and not isinstance(label_child_index, bool) and label_child_index >= -1:
+ relation["label_child_index"] = label_child_index
+ label_selector = _bounded_selector(_structured_str(item.get("label_selector")))
+ if label_selector:
+ relation["label_selector"] = label_selector
+ page_count = item.get("key_text_walked_count")
+ if isinstance(page_count, int) and not isinstance(page_count, bool) and page_count > 0:
+ relation["key_text_walked_count"] = page_count
+ value_count = item.get("value_text_walked_count")
+ if isinstance(value_count, int) and not isinstance(value_count, bool) and value_count > 0:
+ relation["value_text_walked_count"] = value_count
+ if len(str(relation.get("value_text") or "")) >= _MAX_RELATION_VALUE_CHARS:
+ relation["value_truncated"] = True
+ relations.append(relation)
return relations
@@ -2430,7 +2826,7 @@ def _structured_clickable_controls(value: Any) -> list[dict[str, Any]]:
if not isinstance(item, dict):
continue
text = _schema_text(_structured_str(item.get("text")), 120)
- selector = _structured_str(item.get("selector"))[:160]
+ selector = _bounded_selector(_structured_str(item.get("selector")))
entry: dict[str, Any] = {}
if text:
entry["text"] = text
@@ -2457,7 +2853,7 @@ def _structured_challenge_controls(value: Any) -> list[dict[str, Any]]:
"name": _structured_str(node.get("name"))[:120],
"class": _structured_classes(node.get("class")),
"type": _structured_str(node.get("type"))[:40],
- "selector": _structured_str(node.get("selector"))[:160],
+ "selector": _bounded_selector(_structured_str(node.get("selector"))),
"text": _schema_text(_structured_str(node.get("text")), 200),
}
if node.get("checked") is True:
@@ -2484,7 +2880,7 @@ def _structured_modal_dismiss_controls(value: Any) -> list[dict[str, Any]]:
"text": _schema_text(_structured_str(control.get("text")), 120),
"aria_label": _schema_text(_structured_str(control.get("aria_label")), 120),
"title": _schema_text(_structured_str(control.get("title")), 120),
- "selector": _structured_str(control.get("selector"))[:160],
+ "selector": _bounded_selector(_structured_str(control.get("selector"))),
"type": _structured_str(control.get("type"))[:40],
}
controls.append({k: v for k, v in entry.items() if v})
@@ -2503,7 +2899,7 @@ def _structured_modal_overlays(value: Any) -> list[dict[str, Any]]:
"aria_modal": node.get("aria_modal") is True,
"id": _structured_str(node.get("id"))[:120],
"class": _structured_classes(node.get("class")),
- "selector": _structured_str(node.get("selector"))[:160],
+ "selector": _bounded_selector(_structured_str(node.get("selector"))),
"text": _schema_text(_structured_str(node.get("text")), 240),
"dismiss_controls": _structured_modal_dismiss_controls(node.get("dismiss_controls")),
}
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index 9d4c51083..2d01ad74b 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -985,8 +985,6 @@ class CopilotContext(AgentContext):
# construction site, overriding this default.
turn_id: str = field(default_factory=lambda: uuid.uuid4().hex)
turn_index: int = 0
- design_start_emitted: bool = False
- design_end_emitted: bool = False
narrative_summary: str | None = None
staged_workflow_yaml: str | None = None
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index a13f91d0e..71825fd61 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -5,8 +5,10 @@ from __future__ import annotations
import asyncio
import copy
import json
+import os
import re
import time
+import uuid
from collections.abc import Mapping, Sequence
from dataclasses import replace
from typing import TYPE_CHECKING, Any
@@ -108,8 +110,13 @@ from skyvern.forge.sdk.copilot.narration import TransitionKind
from skyvern.forge.sdk.copilot.output_contracts import OutputContractAdvisoryState
from skyvern.forge.sdk.copilot.output_extraction_plan import (
RequestedOutputExtractionPlan,
+ bindable_candidate_headings,
+ derivation_bail_reason,
derive_requested_output_extraction_plan,
+ plan_from_designations,
resolve_shape_expectations_by_path,
+ unbound_candidate_relations,
+ value_shown_in_selectable_evidence,
)
from skyvern.forge.sdk.copilot.output_policy import (
completion_criterion_requires_browser_fill_delivery,
@@ -2091,6 +2098,32 @@ def synthesized_persistence_reopened_after_failed_run(ctx: Any) -> bool:
return _last_scout_interaction_commits(trajectory)
+def synthesized_offer_reopened_for_extraction_plan(
+ ctx: AgentContext, plan: RequestedOutputExtractionPlan | None
+) -> bool:
+ """Whether a newly provable read is worth reopening a closed offer for.
+
+ The offer is made once per authoring, but the value a turn was asked for often only becomes
+ provable after the draft exists — the page carrying it is reached later. Without this the proven
+ read is derived and never offered, and the draft's own guess is what the first run executes.
+ Reopening stops as soon as the authored block already carries this plan, so it cannot loop.
+
+ A page whose wording never matches the request binds no plan at all, and the relations it does
+ offer are named only by the offer this gate guards; requiring a bound plan to reopen therefore
+ asked the read that makes the plan bind to already exist. Unbound candidates reopen it too, and
+ stop doing so once one has been read into a requested path.
+ """
+ if plan is None:
+ requested = _requested_output_paths_for_ctx(ctx)
+ if not requested or not unbound_candidate_relations(ctx.flow_evidence):
+ return False
+ # The offer names relations no read has claimed, so it stops once one of them has been read
+ # into a requested path. Without this the prompt re-carries the whole offer on every build.
+ return not (requested & set(_witness_values_for_derivation(ctx)))
+ carried = (ctx.requested_output_extraction_candidate, ctx.pending_requested_output_extraction_candidate)
+ return all(candidate is None or candidate.plan_identity != plan.identity for candidate in carried)
+
+
def synthesized_persistence_reopened(ctx: AgentContext) -> bool:
if ctx.synthesized_block_reopened_for_credential_scout:
return True
@@ -2286,29 +2319,208 @@ def _effective_requested_output_path(criterion: CompletionCriterion) -> str | No
return None
+def requested_output_paths_for_derivation(ctx: AgentContext) -> set[str]:
+ """The paths derivation will try to bind, including floor-rekeyed and repair-context ones.
+
+ The offer and the binder have to read one set. Gating the offer on the policy's criteria alone
+ withheld it for a path that reached derivation by rekey, so the page's own candidates were never
+ named and the read that would have witnessed the value was never invited (SKY-13226).
+ """
+ return _requested_output_paths_for_ctx(ctx)
+
+
def _requested_output_labels_by_path(ctx: AgentContext) -> dict[str, tuple[str, ...]]:
requested_paths = _requested_output_paths_for_ctx(ctx)
labels_by_path: dict[str, tuple[str, ...]] = {}
for criterion in _pre_run_gated_completion_criteria(ctx):
- outcome = criterion.outcome.strip()
+ label = (criterion.requested_output_label or criterion.outcome).strip()
path = _effective_requested_output_path(criterion)
- if path in requested_paths and outcome:
+ if path in requested_paths and label:
labels_by_path.setdefault(path, ())
- labels_by_path[path] += (outcome,)
+ labels_by_path[path] += (label,)
return labels_by_path
+def _witnessed_values_by_path(ctx: AgentContext) -> dict[str, str]:
+ """The scalar each requested output was read as, keyed by the path the read claimed.
+
+ Capture retains every read of a path and defers the choice to synthesis, so this is where that
+ choice lives: differing reads resolve to the one value a selectable observation still shows,
+ because the page corroborates what was read from it and cannot corroborate a probe's echo. A
+ conflict the page corroborates for none, or for more than one, still carries no witness.
+ """
+ reads: dict[str, list[str]] = {}
+ for interaction in ctx.scout_trajectory:
+ if interaction.get("tool_name") != "read_value":
+ continue
+ # A read that inherited its path by elimination says nothing about that path: an early probe
+ # of a login form was promoted that way and its JSON became the witness for a metric the page
+ # had not shown yet. Only a read that named the path witnesses it (SKY-13226).
+ if interaction.get("read_output_path_source") != "declared":
+ continue
+ path = str(interaction.get("read_output_path") or "")
+ value = str(interaction.get("read_result_value") or "")
+ if path and value:
+ reads.setdefault(path, []).append(value)
+ resolved: dict[str, str] = {}
+ for path, values in reads.items():
+ distinct = list(dict.fromkeys(values))
+ if len(distinct) == 1:
+ resolved[path] = distinct[0]
+ continue
+ shown = [
+ value
+ for value in distinct
+ if value_shown_in_selectable_evidence(getattr(ctx, "flow_evidence", None) or [], value)
+ ]
+ if len(shown) == 1:
+ resolved[path] = shown[0]
+ return resolved
+
+
+def dump_derivation_inputs(ctx: AgentContext, *, outcome: str) -> None:
+ """Write the inputs a derivation was given, when a local run asks for them.
+
+ Derivation and the synthesis it feeds are both pure, so a live outcome is reproducible offline
+ from these values alone; without them every attempt at either costs a full turn. Successes are
+ written as well as failures because the code a bound plan generates is only reachable from a
+ packet that bound. The evidence carries whatever text the page held, so writing takes both an
+ explicit path and a local environment rather than the path alone: a deployed run cannot be
+ talked into dumping page contents by its environment.
+ """
+ directory = os.environ.get("COPILOT_DUMP_DERIVATION_INPUTS")
+ if not directory or settings.ENV != "local":
+ return
+ try:
+ os.makedirs(directory, exist_ok=True)
+ payload = {
+ "outcome": outcome,
+ "labels_by_path": {path: list(labels) for path, labels in _requested_output_labels_by_path(ctx).items()},
+ "witnessed_by_path": _witnessed_values_by_path(ctx),
+ # Scope is authoritative, so a replay that rebuilds it from the labels is not the same
+ # derivation the run performed.
+ "requested_paths": sorted(_requested_output_paths_for_ctx(ctx)),
+ "designations": list(ctx.requested_output_designations),
+ "flow_evidence": ctx.flow_evidence,
+ "scout_trajectory": list(ctx.scout_trajectory),
+ }
+ target = os.path.join(directory, f"derivation-{outcome}-{len(ctx.flow_evidence)}-{uuid.uuid4().hex[:8]}.json")
+ with open(target, "w") as handle:
+ json.dump(payload, handle, default=str)
+ except Exception:
+ LOG.info("copilot_derivation_input_dump_failed", exc_info=True)
+
+
+def _current_page_designations(ctx: AgentContext) -> list[dict[str, Any]]:
+ """Designations pin coordinates on one page; once the browser moves they describe a page the
+ block will not be looking at, so they are dropped rather than compiled into a stale read."""
+ designations = ctx.requested_output_designations
+ if not designations:
+ return []
+ page_evidence = ctx.composition_page_evidence or {}
+ current_url = str(page_evidence.get("current_url") or "")
+ # An unknown current URL is the ordinary state right after designating — the probe stamps the
+ # page, but only a composition inspection records one — so it cannot stand in for staleness.
+ if not current_url:
+ return list(designations)
+ return [
+ designation
+ for designation in designations
+ if not str(designation.get("url") or "") or str(designation.get("url")) == current_url
+ ]
+
+
+def _designated_values_by_path(ctx: AgentContext) -> dict[str, str]:
+ """The value the model designated and the page confirmed, keyed by the path it fills.
+
+ A designation is a witness the page validated against the live DOM, so it arms the value binder
+ without the model having to author an expression that returns exactly one scalar (SKY-13226).
+ """
+ values: dict[str, str] = {}
+ for designation in _current_page_designations(ctx):
+ path = designation.get("output_path")
+ text = designation.get("text")
+ if isinstance(path, str) and isinstance(text, str) and text:
+ values[path] = text
+ return values
+
+
+def _witness_values_for_derivation(ctx: AgentContext) -> dict[str, str]:
+ """Read-witnessed values, overridden for any path the page confirmed a designation for."""
+ return {**_witnessed_values_by_path(ctx), **_designated_values_by_path(ctx)}
+
+
+def _labels_outranked_by_designation(
+ ctx: AgentContext, labels_by_path: dict[str, tuple[str, ...]]
+) -> dict[str, tuple[str, ...]]:
+ """Drop the lexical channel for a path the model designated on the live page.
+
+ Both channels can bind the same path to different relations, and the designation is the one an
+ element was actually resolved from.
+ """
+ designated = _designated_values_by_path(ctx)
+ return {path: labels for path, labels in labels_by_path.items() if path not in designated}
+
+
+def requested_output_extraction_plan_diagnostic(ctx: AgentContext) -> dict[str, Any]:
+ """Why a derivation returned nothing: gate mismatch, or no bindable packet.
+
+ Without both sides plus the trajectory's stamps, an unavailable-plan log cannot separate the two
+ and each guess costs a live run.
+ """
+ requested_paths = _requested_output_paths_for_ctx(ctx)
+ labels_by_path = _requested_output_labels_by_path(ctx)
+ return {
+ "requested_paths": sorted(requested_paths),
+ "label_paths": sorted(labels_by_path),
+ # The values, not just the keys: a fallen-back outcome sentence and a minted page noun have
+ # identical paths, and only the value says which one the binder was actually given.
+ "labels_by_path": {path: list(labels) for path, labels in sorted(labels_by_path.items())},
+ "paths_match": set(labels_by_path) == requested_paths,
+ "designations": len(_current_page_designations(ctx)),
+ "bail_reason": derivation_bail_reason(
+ flow_evidence=ctx.flow_evidence,
+ labels_by_path=labels_by_path,
+ witnessed_by_path=_witness_values_for_derivation(ctx),
+ requested_paths=requested_paths,
+ ),
+ "candidate_headings": bindable_candidate_headings(ctx.flow_evidence),
+ "flow_evidence_reached_via": [
+ str(entry.get("reached_via") or "") for entry in ctx.flow_evidence if isinstance(entry, dict)
+ ],
+ }
+
+
def requested_output_extraction_plan(ctx: AgentContext) -> RequestedOutputExtractionPlan | None:
requested_paths = _requested_output_paths_for_ctx(ctx)
if not requested_paths:
return None
- labels_by_path = _requested_output_labels_by_path(ctx)
- if set(labels_by_path) != requested_paths:
- return None
- return derive_requested_output_extraction_plan(
+ # Labels are one channel for meeting the request, not the definition of it: withholding the plan
+ # unless every requested path carried a label meant a page whose wording the request never uses
+ # was refused before the value witness that exists for it could be tried (SKY-13226).
+ labels_by_path = _labels_outranked_by_designation(ctx, _requested_output_labels_by_path(ctx))
+ plan = derive_requested_output_extraction_plan(
flow_evidence=ctx.flow_evidence,
labels_by_path=labels_by_path,
+ witnessed_by_path=_witness_values_for_derivation(ctx),
+ requested_paths=requested_paths,
)
+ if plan is not None:
+ ctx.last_bound_requested_output_extraction_plan = plan
+ return plan
+ # The structured packet could not carry the designated element — it is truncated, or the value
+ # is not a relation the capture models. The probe already pinned it, so read it where it sits.
+ designated = plan_from_designations(_current_page_designations(ctx), requested_paths)
+ if designated is not None:
+ ctx.last_bound_requested_output_extraction_plan = designated
+ return designated
+ # Derivation reads the freshest packet, and most are truncated or unbindable, so a plan that did
+ # bind every requested path is answered with rather than re-derived away before the imposition
+ # that needs it. A changed request abandons it: it bound paths the turn no longer asks for.
+ retained = ctx.last_bound_requested_output_extraction_plan
+ if retained is not None and set(retained.requested_output_paths) == requested_paths:
+ return retained
+ return None
def requested_scalar_output_extraction_plan(ctx: AgentContext) -> RequestedOutputExtractionPlan | None:
@@ -2325,13 +2537,15 @@ def requested_scalar_output_extraction_plan(ctx: AgentContext) -> RequestedOutpu
if path in requested_paths and outcome:
labels_by_path.setdefault(path, ())
labels_by_path[path] += (outcome,)
- # A path with no label is an underivable field; withholding the plan keeps a clarification legitimate.
- if set(labels_by_path) != requested_paths:
- return None
- return derive_requested_output_extraction_plan(
+ plan = derive_requested_output_extraction_plan(
flow_evidence=ctx.flow_evidence,
- labels_by_path=labels_by_path,
+ labels_by_path=_labels_outranked_by_designation(ctx, labels_by_path),
+ witnessed_by_path=_witness_values_for_derivation(ctx),
+ requested_paths=requested_paths,
)
+ if plan is not None:
+ return plan
+ return plan_from_designations(_current_page_designations(ctx), requested_paths)
def requested_output_extraction_plan_changed(ctx: AgentContext, current: RequestedOutputExtractionPlan | None) -> bool:
@@ -2339,7 +2553,9 @@ def requested_output_extraction_plan_changed(ctx: AgentContext, current: Request
return False
previous = derive_requested_output_extraction_plan(
flow_evidence=ctx.flow_evidence[:-1],
- labels_by_path=_requested_output_labels_by_path(ctx),
+ labels_by_path=_labels_outranked_by_designation(ctx, _requested_output_labels_by_path(ctx)),
+ witnessed_by_path=_witness_values_for_derivation(ctx),
+ requested_paths=_requested_output_paths_for_ctx(ctx),
)
return previous is not None and previous.identity != current.identity
diff --git a/skyvern/forge/sdk/copilot/mcp_adapter.py b/skyvern/forge/sdk/copilot/mcp_adapter.py
index 43ca7e634..12fb1b69f 100644
--- a/skyvern/forge/sdk/copilot/mcp_adapter.py
+++ b/skyvern/forge/sdk/copilot/mcp_adapter.py
@@ -37,6 +37,7 @@ from skyvern.forge.sdk.copilot.blocker_signal import (
from skyvern.forge.sdk.copilot.build_phase import _phase_blocker_signal
from skyvern.forge.sdk.copilot.enforcement import (
register_no_progress_interaction_click,
+ requested_output_paths_for_derivation,
synthesized_block_persistence_signal,
terminal_challenge_blocker_signal_from_current_page_evidence,
)
@@ -68,6 +69,7 @@ _POST_HOOK_CONTEXT_ROLLBACK_FIELDS = (
"pending_browser_interaction_observation",
"scouted_interactions",
"scout_trajectory",
+ "requested_output_designations",
"pending_scout_source_url",
"pending_scout_typed_value",
"pending_scout_role_name",
@@ -129,6 +131,9 @@ class SchemaOverlay:
required_overrides: list[str] | None = None
arg_transforms: dict[str, str] = field(default_factory=dict)
forced_args: dict[str, Any] = field(default_factory=dict)
+ # Params the copilot offers on top of the MCP tool's own schema. A hook consumes them; they are
+ # stripped before the call, so the underlying tool never sees an argument it cannot accept.
+ copilot_params: dict[str, Any] = field(default_factory=dict)
requires_browser: bool = False
timeout: int | None = None
pre_hook: PreHook | None = None
@@ -171,6 +176,37 @@ def _stash_and_emit_current_page_terminal_challenge_blocker(ctx: Any, tool_name:
return payload
+def _requested_output_path_choices(schema: dict[str, Any], paths: list[str]) -> dict[str, Any]:
+ """Present the outputs this turn owes as the choices for the path a read claims.
+
+ A free-form string left the model naming its own purpose, so the read that observed the requested
+ quantity was filed as exploration and the value it saw never witnessed the path the binder owes
+ (SKY-13226). The field stays optional: a read that is genuinely exploration omits it, and every
+ requested path stays available so a later read can refine one already claimed.
+ """
+ properties = schema.get("properties")
+ if not paths or not isinstance(properties, dict):
+ return schema
+ output_path = properties.get("output_path")
+ if not isinstance(output_path, dict):
+ return schema
+ listed = ", ".join(paths)
+ return {
+ **schema,
+ "properties": {
+ **properties,
+ "output_path": {
+ **output_path,
+ "enum": paths,
+ "description": (
+ f"{output_path.get('description', '')} This turn owes: {listed}. Set it to the one "
+ "this read fills; omit it when the read is exploration."
+ ).strip(),
+ },
+ },
+ }
+
+
def _apply_schema_overlay(
input_schema: dict[str, Any],
overlay: SchemaOverlay,
@@ -190,6 +226,8 @@ def _apply_schema_overlay(
required.remove(mcp_param)
required.append(copilot_param)
+ props.update(overlay.copilot_params)
+
if overlay.required_overrides is not None:
required = overlay.required_overrides
@@ -204,7 +242,8 @@ def _transform_args(
arguments: dict[str, Any],
overlay: SchemaOverlay,
) -> dict[str, Any]:
- mcp_args = {k: v for k, v in arguments.items() if k not in overlay.hide_params | _INTERNAL_TOOL_ARG_KEYS}
+ dropped = overlay.hide_params | _INTERNAL_TOOL_ARG_KEYS | frozenset(overlay.copilot_params)
+ mcp_args = {k: v for k, v in arguments.items() if k not in dropped}
for copilot_param, mcp_param in overlay.arg_transforms.items():
if copilot_param in mcp_args:
@@ -407,6 +446,10 @@ class SkyvernOverlayMCPServer(MCPServer):
self._cached_raw_tools = await self._client.list_tools()
raw_tools = self._cached_raw_tools
result: list[MCPTool] = []
+ try:
+ requested_output_path_choices = sorted(requested_output_paths_for_derivation(self._context_provider()))
+ except Exception:
+ requested_output_path_choices = []
for tool in raw_tools:
if tool.name not in self._allowlist:
@@ -417,6 +460,7 @@ class SkyvernOverlayMCPServer(MCPServer):
overlay = self._overlays.get(copilot_name, SchemaOverlay())
schema = _apply_schema_overlay(tool.inputSchema, overlay)
+ schema = _requested_output_path_choices(schema, requested_output_path_choices)
description = overlay.description or tool.description or ""
result.append(
@@ -509,11 +553,33 @@ class SkyvernOverlayMCPServer(MCPServer):
mcp_args["session_id"] = copilot_ctx.browser_session_id
try:
+ # wait_for(timeout=None) is a plain await, so only overlays that declare a ceiling get
+ # one. The ceiling bounds every await under the call — a page evaluate, but also a stale
+ # session handle whose CDP request never answers, which held a turn for 307s (SKY-13226).
if overlay.requires_browser:
async with mcp_browser_context(copilot_ctx):
- raw_result = await self._client.call_tool(mcp_name, mcp_args, raise_on_error=False)
+ raw_result = await asyncio.wait_for(
+ self._client.call_tool(mcp_name, mcp_args, raise_on_error=False),
+ timeout=overlay.timeout,
+ )
else:
- raw_result = await self._client.call_tool(mcp_name, mcp_args, raise_on_error=False)
+ raw_result = await asyncio.wait_for(
+ self._client.call_tool(mcp_name, mcp_args, raise_on_error=False),
+ timeout=overlay.timeout,
+ )
+ except TimeoutError:
+ LOG.warning("MCP tool call timed out", tool=tool_name, ceiling_seconds=overlay.timeout)
+ # The call is cancelled where it stands, so a tool that changes the page may already have
+ # changed it. Reporting a plain failure invites a retry that acts on the page twice.
+ err = {
+ "ok": False,
+ "error": (
+ f"{tool_name} did not answer within {overlay.timeout}s and was cancelled. "
+ "Whether it took effect is unknown; read the page before trying it again."
+ ),
+ }
+ record_tool_step_result_for_ctx(copilot_ctx, tool_name, arguments, err)
+ return _copilot_to_call_tool_result(err)
except Exception as e:
LOG.warning(
"MCP tool call failed",
diff --git a/skyvern/forge/sdk/copilot/output_extraction_plan.py b/skyvern/forge/sdk/copilot/output_extraction_plan.py
index c235082d4..3f155e460 100644
--- a/skyvern/forge/sdk/copilot/output_extraction_plan.py
+++ b/skyvern/forge/sdk/copilot/output_extraction_plan.py
@@ -3,6 +3,7 @@
from __future__ import annotations
import hashlib
+import json
import re
from dataclasses import dataclass
from enum import StrEnum
@@ -61,6 +62,14 @@ class LiveReadBinding:
headers: tuple[str, ...] = ()
row_cell_counts: tuple[int, ...] = ()
row_identities: tuple[str, ...] = ()
+ # Where the label proving this value's identity lives, when it is not the value's sibling.
+ label_selector: str = ""
+ # Which direct child carries that label; a tile may print its figure first.
+ label_child_index: int = 0
+ # Whether the label is what identified this element. A value-witnessed binding chose the element
+ # by the value it showed, so re-reading a heading proves nothing about that choice — and a page
+ # with no heading has none to re-read.
+ identified_by_label: bool = True
@dataclass(frozen=True, slots=True)
@@ -68,7 +77,9 @@ class RequestedOutputExtractionPlan:
requested_output_paths: tuple[str, ...]
observation_step: int
observation_identity: str
- reveal: RevealAnchor
+ # None when the bindings were read off the navigated page; the generated block then replays no
+ # reveal click, because there was none to replay.
+ reveal: RevealAnchor | None
live_reads: tuple[LiveReadBinding, ...]
identity: str
@@ -78,7 +89,7 @@ class FrozenRequestedOutputExtractionCandidate:
plan_identity: str
observation_identity: str
requested_output_paths: tuple[str, ...]
- reveal: RevealAnchor
+ reveal: RevealAnchor | None
interaction_code: str
extraction_code: str
source: str
@@ -129,6 +140,11 @@ def _key_value_bindings(packet: dict[str, Any], labels_by_path: dict[str, tuple[
relations = packet.get("key_value_relations")
if not isinstance(relations, list):
return []
+ # A requested-output label is minted case-folded while the page renders the tile heading
+ # capitalised, so exact equality is tried first and a symmetric fold only rescues its misses.
+ folded_labels_by_path = {
+ path: tuple(entry.casefold() for entry in labels) for path, labels in labels_by_path.items()
+ }
bindings: list[LiveReadBinding] = []
for relation in relations:
if (
@@ -149,9 +165,12 @@ def _key_value_bindings(packet: dict[str, Any], labels_by_path: dict[str, tuple[
continue
if match_count <= position or position < 0 or child_index < 0 or child_count <= child_index:
continue
- output_path = _exact_path(label, labels_by_path)
+ output_path = _exact_path(label, labels_by_path) or _exact_path(label.casefold(), folded_labels_by_path)
if output_path is None:
continue
+ label_selector = relation.get("label_selector")
+ label_selector = label_selector if isinstance(label_selector, str) else ""
+ label_child_index = _relation_label_child_index(relation)
bindings.append(
LiveReadBinding(
output_path,
@@ -162,11 +181,146 @@ def _key_value_bindings(packet: dict[str, Any], labels_by_path: dict[str, tuple[
child_index,
child_count,
relation_label=label,
+ label_selector=label_selector,
+ label_child_index=label_child_index,
+ identified_by_label=label_child_index >= 0 or bool(label_selector),
)
)
return bindings
+def _singly_walked_key_value_bindings(
+ packet: dict[str, Any], labels_by_path: dict[str, tuple[str, ...]], leaf_paths: set[str]
+) -> list[LiveReadBinding]:
+ """Binds a path whose label the extractor's walk saw exactly once.
+
+ Truncation says there is more beyond the capture, which matters only because an unseen relation
+ could share a captured label and make a bind ambiguous. A label the walk met once cannot be that
+ among the relations it walked, so it binds from a truncated channel; everything else still waits
+ for a complete one. Only a single-alias path qualifies: two aliases for one path each counted
+ once would still produce two bindings.
+
+ The count covers the light DOM the extractor walked, not the page: a duplicate inside a shadow
+ root, a cross-origin frame, an unmounted virtualized row, or markup rendered after the capture
+ is not in it, so this is uniqueness among what was seen rather than proof of uniqueness.
+
+ Paths are counted one at a time, which is what counting per label needs and is also blind to the
+ binder's own cross-path check, so two paths naming the same label would each bind that one
+ relation and report one tile as two different outputs; a label more than one path claims binds
+ for none of them.
+ """
+ claimed: dict[str, list[LiveReadBinding]] = {}
+ for path in leaf_paths:
+ labels = labels_by_path.get(path) or ()
+ if len({label.casefold() for label in labels}) != 1:
+ continue
+ candidates = _key_value_bindings(packet, {path: labels})
+ if len(candidates) != 1:
+ continue
+ relation_count = _walked_count_for_label(packet, candidates[0].relation_label)
+ if relation_count != 1:
+ continue
+ claimed.setdefault(candidates[0].relation_label.casefold(), []).append(candidates[0])
+ bindings = [found[0] for found in claimed.values() if len(found) == 1]
+ return bindings
+
+
+def _walked_count_for_label(packet: dict[str, Any], label: str) -> int:
+ relations = packet.get("key_value_relations")
+ if not isinstance(relations, list):
+ return 0
+ folded = label.casefold()
+ for relation in relations:
+ if isinstance(relation, dict) and str(relation.get("key_text") or "").casefold() == folded:
+ count = relation.get("key_text_walked_count")
+ return count if _is_int(count) else 0
+ return 0
+
+
+def _relation_label_child_index(relation: dict[str, Any]) -> int:
+ """Which direct child carries the label, or -1 when the label is not the value's sibling.
+
+ A tile whose figure is re-anchored onto its own row leaves the heading outside that row, so no
+ child index names it and a read proving one would assert the figure against the heading text.
+ """
+ index = relation.get("label_child_index")
+ if not _is_int(index):
+ return 0
+ return index if index >= -1 else 0
+
+
+def _value_witness_bindings(
+ packet: dict[str, Any], witnessed_by_path: dict[str, str], *, channel_intact: bool = True
+) -> list[LiveReadBinding]:
+ """Bind a path to the one relation still showing the value the scout read for it.
+
+ This corroborates one quantity observed through two channels rather than matching a goal word
+ against a page word, so a page whose vocabulary shares nothing with the request still binds.
+ A value that appears in no relation, or in more than one, binds nothing.
+
+ A truncated channel says there is more beyond the capture, so uniqueness among what was recorded
+ is not uniqueness on the page. There the relation must also carry the walk's own page-wide count
+ of its value, and that count must be one; a relation that never carried the count proves nothing
+ and binds nothing. One relation answers at most one requested output, so two paths witnessing the
+ same value describe one observation twice and neither binds.
+ """
+ relations = packet.get("key_value_relations")
+ if not witnessed_by_path or not isinstance(relations, list):
+ return []
+ resolved: list[tuple[int, LiveReadBinding]] = []
+ for output_path, witnessed in witnessed_by_path.items():
+ matches = [
+ relation
+ for relation in relations
+ if isinstance(relation, dict)
+ and relation.get("visible") is True
+ and relation.get("value_visible") is True
+ and relation.get("value_truncated") is not True
+ and isinstance(relation.get("value_text"), str)
+ and relation["value_text"].strip() == witnessed
+ ]
+ if len(matches) != 1:
+ continue
+ relation = matches[0]
+ if not channel_intact and relation.get("value_text_walked_count") != 1:
+ continue
+ selector = relation.get("container_selector")
+ match_count = relation.get("container_match_count")
+ position = relation.get("container_position")
+ child_index = relation.get("value_child_index")
+ child_count = relation.get("direct_child_count")
+ if not isinstance(selector, str) or not selector:
+ continue
+ if not _is_int(match_count) or not _is_int(position) or not _is_int(child_index) or not _is_int(child_count):
+ continue
+ if match_count <= position or position < 0 or child_index < 0 or child_count <= child_index:
+ continue
+ resolved.append(
+ (
+ id(relation),
+ LiveReadBinding(
+ output_path,
+ LiveReadKind.KEY_VALUE,
+ selector,
+ match_count,
+ position,
+ child_index,
+ child_count,
+ relation_label=str(relation.get("key_text") or ""),
+ label_selector=str(relation.get("label_selector") or ""),
+ label_child_index=_relation_label_child_index(relation),
+ identified_by_label=False,
+ ),
+ )
+ )
+ claimed_once = {
+ relation_id
+ for relation_id in {rid for rid, _ in resolved}
+ if [rid for rid, _ in resolved].count(relation_id) == 1
+ }
+ return [binding for relation_id, binding in resolved if relation_id in claimed_once]
+
+
def _table_bindings(packet: dict[str, Any], labels_by_path: dict[str, tuple[str, ...]]) -> list[LiveReadBinding]:
containers = packet.get("result_containers")
if not isinstance(containers, list):
@@ -510,34 +664,132 @@ def _table_shape_bindings(
return bindings
+# Content a click revealed is unreproducible without replaying that click, so an interaction-reached
+# packet still owes its anchor. Content already rendered on the page owes nothing: requiring an anchor
+# there leaves the plan underived on every deep-linked dashboard (SKY-13226). `navigate` is the stamp for
+# an inspection that went to the requested URL itself, which is exactly the deep-link case; `post_run` is
+# excluded because it observes the state a run left behind rather than the page as authored.
+_BINDABLE_REACHED_VIA: frozenset[str] = frozenset({"interaction", "current_page", "navigate"})
+_PAGE_EVIDENCE_TOOLS: frozenset[str] = frozenset({"inspect_page_for_composition", "evaluate"})
+_BINDABLE_EVIDENCE_TOOLS: dict[str, frozenset[str]] = {
+ "interaction": frozenset({"scout_interaction"}),
+ "current_page": _PAGE_EVIDENCE_TOOLS,
+ "navigate": _PAGE_EVIDENCE_TOOLS,
+}
+
+
+def _reveal_anchor_from_packet(packet: dict[str, Any]) -> RevealAnchor | None:
+ selector, role, name = (
+ packet.get("interaction_selector"),
+ packet.get("interaction_role"),
+ packet.get("interaction_accessible_name"),
+ )
+ if isinstance(selector, str) and selector:
+ return RevealAnchor(selector=selector)
+ if isinstance(role, str) and role and isinstance(name, str) and name:
+ return RevealAnchor(role=role, name=name)
+ return None
+
+
+def _entry_is_bindable(entry: object) -> bool:
+ """Whether an entry is one derivation can spend its single attempt on.
+
+ The binder needs a bounded schema, so an entry without one carries nothing to bind and choosing
+ it only spends the attempt; selecting on `reached_via` alone let that happen. An observation whose
+ every relation belongs to a dialog in front of the page is the same kind of non-packet: it is
+ freshest, so it wins the single attempt and shadows the tile captured before the dialog opened.
+ """
+ return _entry_observed_the_page(entry) and entry.get("had_bounded_schema") is True
+
+
+def _entry_observed_the_page(entry: object) -> TypeGuard[dict[str, Any]]:
+ """Whether the entry is an observation of the page itself, so a walk may select it at all.
+
+ Three walks each pick "the freshest bindable packet", so a dialog-only capture excluded from one
+ of them is still chosen by the other two; they share this test rather than repeating its terms.
+ """
+ return (
+ isinstance(entry, dict)
+ and entry.get("reached_via") in _BINDABLE_REACHED_VIA
+ and entry.get("obstructed") is not True
+ )
+
+
+def _intact_binding_channels(packet: dict[str, Any]) -> tuple[bool, bool]:
+ """Whether the key/value and result-container channels each survived the capture caps.
+
+ A capture that hit one cap says nothing about the other, and a key/value bind reads only
+ relations while a table bind reads only containers, so vetoing the whole packet withholds a
+ binding whose own evidence is complete. Truncation still voids the channel it happened in,
+ where an unseen duplicate could have made the bind ambiguous.
+ """
+ return (
+ packet.get("key_value_relations_truncated") is False,
+ packet.get("result_containers_truncated") is False,
+ )
+
+
+def _channel_bindings(
+ packet: dict[str, Any],
+ labels_by_path: dict[str, tuple[str, ...]],
+ *,
+ key_values_intact: bool,
+ containers_intact: bool,
+) -> list[LiveReadBinding]:
+ bindings: list[LiveReadBinding] = []
+ if key_values_intact:
+ bindings += _key_value_bindings(packet, labels_by_path)
+ if containers_intact:
+ bindings += _table_bindings(packet, labels_by_path)
+ return bindings
+
+
def _plan_from_entry(
- entry: dict[str, Any], *, labels_by_path: dict[str, tuple[str, ...]]
+ entry: dict[str, Any],
+ *,
+ labels_by_path: dict[str, tuple[str, ...]],
+ witnessed_by_path: dict[str, str] | None = None,
+ requested_paths: set[str] | None = None,
) -> RequestedOutputExtractionPlan | None:
- if entry.get("reached_via") != "interaction" or entry.get("had_bounded_schema") is not True:
+ reached_via = entry.get("reached_via")
+ if reached_via not in _BINDABLE_REACHED_VIA or entry.get("had_bounded_schema") is not True:
return None
step, packet = entry.get("step"), entry.get("evidence")
- if not _is_int(step) or not isinstance(packet, dict) or packet.get("source_tool") != "scout_interaction":
+ if not _is_int(step) or not isinstance(packet, dict):
return None
- if (
- packet.get("result_containers_truncated") is not False
- or packet.get("key_value_relations_truncated") is not False
- ):
+ if packet.get("source_tool") not in _BINDABLE_EVIDENCE_TOOLS[reached_via]:
return None
+ key_values_intact, containers_intact = _intact_binding_channels(packet)
if isinstance(packet.get("inspection_warnings"), list) and packet["inspection_warnings"]:
return None
- selector, role, name = (
- packet.get("interaction_selector"),
- packet.get("interaction_role"),
- packet.get("interaction_accessible_name"),
- )
- if isinstance(selector, str) and selector:
- reveal = RevealAnchor(selector=selector)
- elif isinstance(role, str) and role and isinstance(name, str) and name:
- reveal = RevealAnchor(role=role, name=name)
- else:
+ reveal = _reveal_anchor_from_packet(packet)
+ if reveal is None and reached_via == "interaction":
return None
- leaf_paths = _leaf_paths(set(labels_by_path))
- live_reads = _key_value_bindings(packet, labels_by_path) + _table_bindings(packet, labels_by_path)
+ requested_scope = set(requested_paths) if requested_paths else set(labels_by_path)
+ leaf_paths = _leaf_paths(requested_scope)
+ live_reads = _channel_bindings(
+ packet,
+ labels_by_path,
+ key_values_intact=key_values_intact,
+ containers_intact=containers_intact,
+ )
+ if not key_values_intact:
+ # The channel stays truncated: a label counted once page-wide binds, and nothing else does.
+ live_reads = live_reads + _singly_walked_key_value_bindings(packet, labels_by_path, leaf_paths)
+ # The witness joins on the value, and refuses a value the page shows more than once, so it carries
+ # its own ambiguity proof rather than borrowing the channel's. Requiring an intact channel switched
+ # it off for every dashboard rich enough to trip the relation cap.
+ if witnessed_by_path:
+ label_bound = {binding.output_path for binding in live_reads}
+ live_reads = live_reads + _value_witness_bindings(
+ packet,
+ {
+ path: value
+ for path, value in witnessed_by_path.items()
+ if path in leaf_paths and path not in label_bound
+ },
+ channel_intact=key_values_intact,
+ )
by_path: dict[str, list[LiveReadBinding]] = {}
for binding in live_reads:
by_path.setdefault(binding.output_path, []).append(binding)
@@ -554,19 +806,328 @@ def _plan_from_entry(
# The step is provenance, not structure: an identical re-observation must not
# invalidate the candidate frozen from the earlier offer.
observation_identity = _stable_identity(repr((reveal, ordered_reads)))
- identity = _stable_identity(repr((tuple(sorted(labels_by_path)), observation_identity)))
- return RequestedOutputExtractionPlan(
- tuple(sorted(labels_by_path)), step, observation_identity, reveal, ordered_reads, identity
+ # The scope the plan was derived for, not the labels that happened to bind it: a witness binds
+ # without a label, and keying either field on labels leaves the plan claiming no requested path.
+ requested = tuple(sorted(requested_scope))
+ identity = _stable_identity(repr((requested, observation_identity)))
+ return RequestedOutputExtractionPlan(requested, step, observation_identity, reveal, ordered_reads, identity)
+
+
+def value_designation_probe_expression(value_text: str, label: str) -> str:
+ """In-browser probe resolving a value the model read off the page into a pinned element.
+
+ The model designates what it can see — the rendered value, and the label it sits under — and the
+ page resolves that to a selector. Asking the model for markup instead made it author selectors it
+ had no way to verify (SKY-13226).
+ """
+ target = json.dumps(value_text)
+ anchor = json.dumps(label)
+ return (
+ "(() => { const target = " + target + "; const label = " + anchor + ";"
+ " const visible = (el) => { const r = el.getBoundingClientRect();"
+ " return r.width > 0 && r.height > 0; };"
+ " let matches = Array.from(document.querySelectorAll('body *')).filter((el) => {"
+ " if (!visible(el)) return false;"
+ " if ((el.innerText || '').trim() !== target) return false;"
+ " return !Array.from(el.children).some((c) => (c.innerText || '').trim() === target); });"
+ " if (!matches.length) return { error: 'text-not-found' };"
+ " if (matches.length > 1 && label) { const scoped = matches.filter((el) => {"
+ " let node = el.parentElement, hops = 0;"
+ " while (node && hops < 4) { if ((node.innerText || '').includes(label)) return true;"
+ " node = node.parentElement; hops++; } return false; });"
+ " if (scoped.length) matches = scoped; }"
+ " if (matches.length > 1) return { error: 'text-ambiguous', visible_count: matches.length,"
+ " text: target, url: location.href };"
+ " const chosen = matches[0];"
+ # Identity before position: a positional path silently reads the wrong element once a sibling
+ # is inserted between designation and the block running.
+ " const unique = (sel) => { try { const found = document.querySelectorAll(sel);"
+ " return found.length === 1 && found[0] === chosen; } catch (e) { return false; } };"
+ " const tag = chosen.tagName.toLowerCase();"
+ " const candidates = [];"
+ " if (chosen.id) candidates.push('#' + CSS.escape(chosen.id));"
+ " const classes = Array.from(chosen.classList || []).slice(0, 3)"
+ ".map((c) => '.' + CSS.escape(c)).join('');"
+ " if (classes) candidates.push(tag + classes);"
+ " let selector = '';"
+ " for (const candidate of candidates) { if (unique(candidate)) { selector = candidate; break; } }"
+ " if (!selector) { const parts = []; let cur = chosen;"
+ " while (cur && cur.nodeType === 1 && cur !== document.body) {"
+ " const parent = cur.parentElement; if (!parent) break;"
+ " const index = Array.prototype.indexOf.call(parent.children, cur) + 1;"
+ " parts.unshift(cur.tagName.toLowerCase() + ':nth-child(' + index + ')');"
+ " if (parent.id) { parts.unshift('#' + CSS.escape(parent.id));"
+ " cur = null; break; } cur = parent; }"
+ " selector = parts[0] && parts[0].charAt(0) === '#'"
+ " ? parts.join(' > ') : 'body > ' + parts.join(' > '); }"
+ " const all = Array.from(document.querySelectorAll(selector));"
+ " const position = all.indexOf(chosen);"
+ " if (position < 0) return { error: 'path-unstable', text: target, url: location.href };"
+ " return { selector: selector, match_count: all.length, position: position, text: target,"
+ " url: location.href }; })()"
)
+def plan_from_designations(
+ designations: list[dict[str, Any]], requested_paths: set[str]
+) -> RequestedOutputExtractionPlan | None:
+ """Build the extraction plan from model-designated, probe-validated value elements.
+
+ Designation replaces shape inference as the chooser: the page pins and compiles what the model
+ saw, rather than the model authoring a path to an element it can only see rendered (SKY-13226).
+ Independent of packet completeness — validation ran against the live page, not a truncatable
+ relation list.
+ """
+ leaf_paths = _leaf_paths(requested_paths)
+ if not leaf_paths:
+ return None
+ by_path: dict[str, LiveReadBinding] = {}
+ for designation in designations:
+ path = designation.get("output_path")
+ selector = designation.get("selector")
+ match_count = designation.get("match_count")
+ position = designation.get("position")
+ if not isinstance(path, str) or path not in leaf_paths or path in by_path:
+ return None
+ # A designation the page saw but could not pin to one element carries no selector. It still
+ # names the value, which the witness channel binds; here it simply leaves the path uncovered.
+ if not isinstance(selector, str) or not selector:
+ continue
+ if not _is_int(match_count) or not _is_int(position) or match_count <= position or position < 0:
+ return None
+ by_path[path] = LiveReadBinding(path, LiveReadKind.KEY_VALUE, selector, match_count, position)
+ if set(by_path) != leaf_paths:
+ return None
+ ordered_reads = tuple(by_path[path] for path in sorted(leaf_paths))
+ observation_identity = _stable_identity(repr(("designated", ordered_reads)))
+ requested = tuple(sorted(requested_paths))
+ identity = _stable_identity(repr((requested, observation_identity)))
+ return RequestedOutputExtractionPlan(requested, 0, observation_identity, None, ordered_reads, identity)
+
+
+def derivation_bail_reason(
+ *,
+ flow_evidence: list[dict[str, Any]],
+ labels_by_path: dict[str, tuple[str, ...]],
+ witnessed_by_path: dict[str, str] | None = None,
+ requested_paths: set[str] | None = None,
+) -> str:
+ """Which guard stopped derivation, for the unavailable-plan log.
+
+ Ten live runs produced `derived=0` with the tile visibly on screen, and each cause took a
+ ~15-minute run plus a guess to distinguish; the bail point makes the next run name it. A missing
+ label stopped being terminal once a witness could bind without one, so the label channel and the
+ witness channel report separately rather than both as "no-labels".
+ """
+ scope = set(requested_paths) if requested_paths else set(labels_by_path)
+ if not scope:
+ return "no-authoritative-paths"
+ # Selected the way derivation selects, or the reason names an entry derivation never looked at.
+ entry = next((candidate for candidate in reversed(flow_evidence) if _entry_is_bindable(candidate)), None)
+ if entry is None:
+ return "no-bindable-entry"
+ reached_via = str(entry.get("reached_via"))
+ if entry.get("had_bounded_schema") is not True:
+ return f"entry-unbounded-schema[{reached_via}]"
+ packet = entry.get("evidence")
+ if not _is_int(entry.get("step")) or not isinstance(packet, dict):
+ return f"entry-malformed[{reached_via}]"
+ source_tool = str(packet.get("source_tool"))
+ if source_tool not in _BINDABLE_EVIDENCE_TOOLS[reached_via]:
+ return f"packet-source-tool[{reached_via}:{source_tool}]"
+ key_values_intact, containers_intact = _intact_binding_channels(packet)
+ voided_channels = [
+ name
+ for name, intact in (
+ ("key_value_relations", key_values_intact),
+ ("result_containers", containers_intact),
+ )
+ if not intact
+ ]
+ # Truncation stopped being terminal once a witness could bind through it, so it is reported as
+ # context on the per-path detail below rather than as the answer.
+ if isinstance(packet.get("inspection_warnings"), list) and packet["inspection_warnings"]:
+ return f"packet-inspection-warnings[{reached_via}]"
+ if reached_via == "interaction" and _reveal_anchor_from_packet(packet) is None:
+ return f"missing-reveal-anchor[{source_tool}]"
+ leaf_paths = _leaf_paths(scope)
+ counts = {path: 0 for path in leaf_paths}
+ for binding in _channel_bindings(
+ packet,
+ labels_by_path,
+ key_values_intact=key_values_intact,
+ containers_intact=containers_intact,
+ ):
+ if binding.output_path in counts:
+ counts[binding.output_path] += 1
+ for binding in _value_witness_bindings(packet, dict(witnessed_by_path or {}), channel_intact=key_values_intact):
+ if binding.output_path in counts:
+ counts[binding.output_path] += 1
+ unbound = sorted(path for path, count in counts.items() if count == 0)
+ ambiguous = sorted(f"{path}(n={count})" for path, count in counts.items() if count > 1)
+ if unbound:
+ # An unbound path names why its witness channel could not answer, because a vocabulary miss,
+ # an undeclared read and an unproved count want three different fixes.
+ witnessed = dict(witnessed_by_path or {})
+ relations = [rel for rel in (packet.get("key_value_relations") or []) if isinstance(rel, dict)]
+ detail = []
+ for path in unbound:
+ value = witnessed.get(path)
+ if not value:
+ detail.append(f"{path}:witness-not-declared")
+ continue
+ matches = [rel for rel in relations if str(rel.get("value_text") or "").strip() == value]
+ if not matches:
+ detail.append(f"{path}:witness-not-present")
+ elif len(matches) > 1:
+ detail.append(f"{path}:witness-ambiguous[n={len(matches)}]")
+ elif not key_values_intact and matches[0].get("value_text_walked_count") != 1:
+ detail.append(f"{path}:witness-uniqueness-unproved-after-cap")
+ else:
+ detail.append(f"{path}:requested-path-unbound")
+ truncated = f" truncated={voided_channels}" if voided_channels else ""
+ return f"bindings[{' '.join(detail)}{(' ambiguous=' + str(ambiguous)) if ambiguous else ''}{truncated}]"
+ if ambiguous:
+ truncated = f" truncated={voided_channels}" if voided_channels else ""
+ return f"bindings[ambiguous={ambiguous}{truncated}]"
+ return "table-consistency-or-derived"
+
+
+def bindable_candidate_headings(flow_evidence: list[dict[str, Any]], *, limit: int = 8) -> list[str]:
+ """Key texts of the freshest bindable packet, for the unavailable-plan log.
+
+ The binder compares minted labels against these; without both sides in the log, a live no-bind
+ cannot say whether the label or the page vocabulary missed.
+ """
+ for entry in reversed(flow_evidence):
+ if not _entry_observed_the_page(entry):
+ continue
+ packet = entry.get("evidence")
+ if not isinstance(packet, dict):
+ return []
+ relations = packet.get("key_value_relations")
+ if not isinstance(relations, list):
+ return []
+ return [
+ str(relation.get("key_text"))
+ for relation in relations[:limit]
+ if isinstance(relation, dict) and relation.get("key_text")
+ ]
+ return []
+
+
+def unbound_candidate_relations(flow_evidence: list[dict[str, Any]], *, limit: int = 8) -> list[tuple[str, str]]:
+ """Label/value pairs the freshest bindable packet is offering.
+
+ The binder joins a minted label to a page label, so a page that names the requested quantity in
+ its own words binds nothing however plainly it displays it. Handing the loop what the page does
+ offer lets it read one of these into the requested path, after which the value witness binds the
+ relation still showing that exact value — a join on the quantity rather than on the wording.
+ """
+ for entry in reversed(flow_evidence):
+ if not _entry_observed_the_page(entry):
+ continue
+ packet = entry.get("evidence")
+ if not isinstance(packet, dict):
+ return []
+ relations = packet.get("key_value_relations")
+ if not isinstance(relations, list):
+ return []
+ # A dialog beside readable content keeps the packet composable, but its own dismiss controls
+ # are not quantities the page is showing; offering them puts a button next to the number as
+ # if they were alternatives for the same requested output.
+ dismiss_texts = set(entry.get("dismiss_texts") or ())
+ offered: list[tuple[str, str]] = []
+ for relation in relations:
+ if not isinstance(relation, dict) or relation.get("visible") is not True:
+ continue
+ if relation.get("value_visible") is not True or relation.get("value_truncated") is True:
+ continue
+ label = str(relation.get("key_text") or "").strip()
+ value = str(relation.get("value_text") or "").strip()
+ if label in dismiss_texts or value in dismiss_texts:
+ continue
+ if label and value and (label, value) not in offered:
+ offered.append((label, value))
+ if len(offered) >= limit:
+ break
+ return offered
+ return []
+
+
+def value_shown_in_selectable_evidence(flow_evidence: list[dict[str, Any]], value: str) -> bool:
+ """Whether an observation selection may consider still shows this exact value."""
+ needle = value.strip()
+ if not needle:
+ return False
+ for entry in reversed(flow_evidence):
+ if not _entry_is_bindable(entry):
+ continue
+ packet = entry.get("evidence")
+ if not isinstance(packet, dict):
+ continue
+ for relation in packet.get("key_value_relations") or []:
+ if isinstance(relation, dict) and str(relation.get("value_text") or "").strip() == needle:
+ return True
+ return False
+
+
+def _witnessed_packet_entry(
+ flow_evidence: list[dict[str, Any]], witnessed_by_path: dict[str, str]
+) -> dict[str, Any] | None:
+ """Newest selectable entry whose packet still shows one of the witnessed values."""
+ witnessed = {value.strip() for value in witnessed_by_path.values() if value and value.strip()}
+ if not witnessed:
+ return None
+ for entry in reversed(flow_evidence):
+ if not _entry_is_bindable(entry):
+ continue
+ packet = entry.get("evidence")
+ if not isinstance(packet, dict):
+ continue
+ for relation in packet.get("key_value_relations") or []:
+ if isinstance(relation, dict) and str(relation.get("value_text") or "").strip() in witnessed:
+ return entry
+ return None
+
+
def derive_requested_output_extraction_plan(
- *, flow_evidence: list[dict[str, Any]], labels_by_path: dict[str, tuple[str, ...]]
+ *,
+ flow_evidence: list[dict[str, Any]],
+ labels_by_path: dict[str, tuple[str, ...]],
+ witnessed_by_path: dict[str, str] | None = None,
+ requested_paths: set[str] | None = None,
) -> RequestedOutputExtractionPlan | None:
- """Derive from one rollback-owned packet; never combine partial observations."""
- if not labels_by_path:
+ """Derive from one rollback-owned packet; never combine partial observations.
+
+ The requested paths define what the plan owes; labels are one channel for meeting it. Scoping the
+ plan by labels instead made a label-free witness depend on a label, so a page whose wording the
+ request never uses had no route at all (SKY-13226).
+ """
+ scope = set(requested_paths) if requested_paths else set(labels_by_path)
+ if not scope:
return None
+ # Freshest bindable packet wins, whether it was reached by a click or was already on the page: a
+ # login-gated dashboard stamps its post-login captures `current_page`, so scanning for `interaction`
+ # alone walks back past the metric tiles into the login form and derives nothing. Falling back to a
+ # staler packet on failure is the partial-observation combining this function refuses; the one other
+ # entry ever tried is the packet still showing a witnessed value, because a live counter ticks
+ # between the read and the freshest capture, and the packet showing what the read saw is that
+ # read's own contemporaneous observation, not a stale substitute. Either way one packet decides.
for entry in reversed(flow_evidence):
- if isinstance(entry, dict) and entry.get("reached_via") == "interaction":
- return _plan_from_entry(entry, labels_by_path=labels_by_path)
+ if _entry_is_bindable(entry):
+ plan = _plan_from_entry(
+ entry, labels_by_path=labels_by_path, witnessed_by_path=witnessed_by_path, requested_paths=scope
+ )
+ if plan is not None or not witnessed_by_path:
+ return plan
+ witness_entry = _witnessed_packet_entry(flow_evidence, witnessed_by_path)
+ if witness_entry is not None and witness_entry is not entry:
+ return _plan_from_entry(
+ witness_entry,
+ labels_by_path=labels_by_path,
+ witnessed_by_path=witnessed_by_path,
+ requested_paths=scope,
+ )
+ return None
return None
diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.py
index 84061f354..192bd93bc 100644
--- a/skyvern/forge/sdk/copilot/request_policy.py
+++ b/skyvern/forge/sdk/copilot/request_policy.py
@@ -447,6 +447,9 @@ class CompletionCriterion:
# YAML; "run": an end state only a run can evidence. Invalid input coerces to "run".
level: CriterionLevel = "run"
output_path: str | None = None
+ # Short page-facing noun for ``output_path`` (e.g. "visitors"), carried typed so the
+ # extraction binder never has to re-tokenize it back out of ``outcome`` prose.
+ requested_output_label: str | None = None
expected_output_value: ExpectedOutputValue | None = None
expected_output_shape: ExpectedOutputShape | None = None
requested_output_evidence_source: RequestedOutputEvidenceSource = "runtime_output"
@@ -3364,6 +3367,7 @@ def _apply_requested_output_completion_criteria(
outcome=f"The returned record includes {field_label}.",
level="run",
output_path=output_path,
+ requested_output_label=field_label,
expected_output_value=expected_value,
expected_output_shape=expected_shape,
requested_output_evidence_source=source_by_output_path.get(output_path, "runtime_output"),
@@ -3391,6 +3395,72 @@ def _apply_requested_output_completion_criteria(
)
+def _detected_output_candidates(user_message: str, aliases: dict[str, str] | None = None) -> tuple[str, ...]:
+ """Requested-output fields the deterministic scan finds, for the slot producer's bind-or-refute
+ obligation. Verbatim message substrings only — the detector composes phrases across spans, and
+ an obligation the producer cannot quote could only ever be refuted."""
+ message_fold = user_message.casefold()
+ seen: set[str] = set()
+ candidates: list[str] = []
+ for field_name in _requested_output_fields(user_message, _normalize_requested_output_aliases(aliases)):
+ cleaned = field_name.strip()
+ fold = cleaned.casefold()
+ if not cleaned or len(cleaned) > 64 or fold in seen or fold not in message_fold:
+ continue
+ seen.add(fold)
+ candidates.append(cleaned)
+ if len(candidates) == 8:
+ break
+ return tuple(candidates)
+
+
+def _apply_requested_output_labels(
+ policy: RequestPolicy, user_message: str, aliases: dict[str, str] | None = None
+) -> None:
+ """Stamp the short requested-output noun onto classifier-authored criteria by ``output_path``,
+ which the canonical mint already carries but classifier-authored criteria only name in prose."""
+
+ schema_aliases = _normalize_requested_output_aliases(
+ schema_output_path_aliases_from_criteria(policy.completion_criteria)
+ )
+ config_aliases = _normalize_requested_output_aliases(aliases)
+ label_by_output_path: dict[str, str] = {}
+ label_by_field_name: dict[str, str] = {}
+ for field_name in _requested_output_fields(user_message, {**config_aliases, **schema_aliases}):
+ output_path = _requested_output_path_for_detected_field(field_name, schema_aliases, config_aliases)
+ label = _requested_output_field_label(field_name, output_path)
+ label_by_output_path.setdefault(output_path, label)
+ label_by_field_name.setdefault(field_name, label)
+ if not label_by_output_path:
+ return
+ policy.completion_criteria = [
+ replace(criterion, requested_output_label=label_by_output_path[criterion.output_path])
+ if criterion.requested_output_label is None
+ and criterion.output_path is not None
+ and criterion.output_path in label_by_output_path
+ else criterion
+ for criterion in policy.completion_criteria
+ ]
+ # The path join misses whenever the classifier names the field differently than the detector
+ # derives it ("visitor_count" vs "visitors"); the criterion's own outcome text still names the
+ # field, so a uniquely covering criterion takes the label rather than falling back to prose.
+ for field_name, label in label_by_field_name.items():
+ covering = [
+ index
+ for index, criterion in enumerate(policy.completion_criteria)
+ if criterion.requested_output_label is None
+ and criterion.output_path is not None
+ and _criterion_text_covers_requested_output(criterion, field_name)
+ ]
+ if len(covering) != 1:
+ continue
+ index = covering[0]
+ policy.completion_criteria = [
+ replace(criterion, requested_output_label=label) if position == index else criterion
+ for position, criterion in enumerate(policy.completion_criteria)
+ ]
+
+
def _validation_classification_target_for_legacy_criterion(
criterion: CompletionCriterion,
) -> tuple[str, ClassificationTarget] | None:
@@ -3678,6 +3748,7 @@ def _classifier_fallback_policy(
_apply_requested_output_completion_criteria(
policy, user_message, requested_output_path_aliases, extract_literals=True
)
+ _apply_requested_output_labels(policy, user_message, requested_output_path_aliases)
_apply_classifier_typed_requested_output_corroborators(policy)
_mark_turn_unsatisfiable_fallback_criteria(policy)
_degrade_pathless_contingent_criteria(policy)
@@ -4024,6 +4095,7 @@ async def _classify_request(
() if transcript.retained_history == _EMPTY_SLOT_SENTINEL else (transcript.retained_history,)
),
global_context=safe_global_llm_context[:32_768],
+ detected_output_candidates=_detected_output_candidates(safe_user_message, requested_output_path_aliases),
)
prompt = prompt_engine.load_prompt(
template=PROMPT_NAME,
@@ -4213,6 +4285,14 @@ async def _classify_request(
if request_slot_result.status == "success":
request_slot_contract = request_slot_result.contract
request_slot_contract_request = request_slot_request
+ if request_slot_contract is not None and request_slot_contract.candidate_refutations:
+ LOG.info(
+ "request-policy producer refuted detected output candidates",
+ refutations=[
+ {"candidate": refutation.candidate, "reason": refutation.reason}
+ for refutation in request_slot_contract.candidate_refutations
+ ],
+ )
else:
request_slot_failure_kind = (
request_slot_result.failure_kind.value if request_slot_result.failure_kind is not None else "unknown"
@@ -4339,6 +4419,7 @@ async def _classify_request(
policy.requires_user_clarification = False
policy.raw_secret_evidence = None
_degrade_pathless_contingent_criteria(policy)
+ _apply_requested_output_labels(policy, user_message, requested_output_path_aliases)
await _reconcile_missing_terminal_action(
policy,
user_message=safe_user_message,
diff --git a/skyvern/forge/sdk/copilot/request_slots.py b/skyvern/forge/sdk/copilot/request_slots.py
index 1f8dc5142..2a2cd5767 100644
--- a/skyvern/forge/sdk/copilot/request_slots.py
+++ b/skyvern/forge/sdk/copilot/request_slots.py
@@ -91,6 +91,19 @@ class RequestSlotProducerInputV1(BaseModel):
retained_history: tuple[str, ...] = Field(max_length=8)
global_context: str = Field(max_length=32_768)
datum_targets: tuple[RequestSlotDatumTargetV1, ...] = Field(default=(), max_length=64)
+ # Requested-output fields the deterministic detector found in the message. Each obliges the
+ # producer to anchor a slot on it or refute it by name; silence stops being a valid answer
+ # for a field the message demonstrably requests (SKY-13329).
+ detected_output_candidates: tuple[str, ...] = Field(default=(), max_length=8)
+
+ @field_validator("detected_output_candidates")
+ @classmethod
+ def _validate_detected_output_candidates(cls, candidates: tuple[str, ...]) -> tuple[str, ...]:
+ if any(not candidate.strip() or len(candidate) > 64 for candidate in candidates):
+ raise ValueError("detected output candidates must be non-empty and at most 64 characters")
+ if len({candidate.casefold() for candidate in candidates}) != len(candidates):
+ raise ValueError("detected output candidates must be unique")
+ return candidates
@field_validator("retained_history")
@classmethod
@@ -166,6 +179,13 @@ class RequestSlotDatumDeclineDeclarationV1(BaseModel):
declined: Literal[True]
+class RequestSlotCandidateRefutationV1(BaseModel):
+ model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
+
+ candidate: str = Field(min_length=1, max_length=64)
+ reason: str = Field(min_length=1, max_length=256)
+
+
class RequestSlotEnvelopeV1(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
@@ -174,12 +194,14 @@ class RequestSlotEnvelopeV1(BaseModel):
datum_bindings: tuple[RequestSlotDatumBindingDeclarationV1 | RequestSlotDatumDeclineDeclarationV1, ...] = Field(
default=(), max_length=64
)
+ candidate_refutations: tuple[RequestSlotCandidateRefutationV1, ...] = Field(default=(), max_length=8)
def _request_digest(
version: str,
sources: tuple[RequestSlotSourceV1, ...],
datum_targets: tuple[RequestSlotDatumTargetV1, ...],
+ detected_output_candidates: tuple[str, ...] = (),
) -> str:
digest_payload: list[object] = [
version,
@@ -189,6 +211,8 @@ def _request_digest(
# targeted producer contracts identity-complete for replay and caching.
if datum_targets:
digest_payload.append([target.model_dump(mode="json") for target in datum_targets])
+ if detected_output_candidates:
+ digest_payload.append(list(detected_output_candidates))
encoded = json.dumps(
digest_payload,
ensure_ascii=True,
@@ -198,7 +222,9 @@ def _request_digest(
def request_slot_request_digest(request: RequestSlotProducerInputV1) -> str:
- return _request_digest(request.version, request_slot_sources(request), request.datum_targets)
+ return _request_digest(
+ request.version, request_slot_sources(request), request.datum_targets, request.detected_output_candidates
+ )
_CANONICAL_SLOT_LEAF_PREFIX = "request_slot_"
@@ -289,6 +315,7 @@ class RequestSlotContractV1(BaseModel):
count: int = Field(ge=0, le=64)
datum_bindings: tuple[CanonicalRequestSlotDatumBindingV1, ...] = Field(default=(), max_length=64)
datum_declines: tuple[CanonicalRequestSlotDatumDeclineV1, ...] = Field(default=(), max_length=64)
+ candidate_refutations: tuple[RequestSlotCandidateRefutationV1, ...] = Field(default=(), max_length=8)
@model_validator(mode="after")
def _validate_derived_membership(self) -> RequestSlotContractV1:
@@ -484,6 +511,33 @@ def canonicalize_request_slots(
raise ValueError(f"source anchors overlap in {item[1].source_id}")
previous = item
+ # Mirror of the datum-target coverage rule above: every detector candidate is answered — a slot
+ # anchored on it, or a refutation naming it. An envelope silent on a candidate is invalid output.
+ candidates_by_fold = {candidate.casefold(): candidate for candidate in request.detected_output_candidates}
+ refuted_folds: set[str] = set()
+ for refutation in envelope.candidate_refutations:
+ fold = refutation.candidate.casefold()
+ if fold not in candidates_by_fold:
+ raise ValueError(f"refutation names an undetected candidate: {refutation.candidate}")
+ if fold in refuted_folds:
+ raise ValueError(f"candidate refuted more than once: {refutation.candidate}")
+ refuted_folds.add(fold)
+ answer_quotes = [declaration.source_quote.casefold() for declaration, _, _, _ in resolved] + [
+ binding.source_quote.casefold()
+ for binding in envelope.datum_bindings
+ if isinstance(binding, RequestSlotDatumBindingDeclarationV1)
+ ]
+ # Either containment direction answers: the detector composes phrases wider than the tile of
+ # text a producer legitimately quotes ("adjacent account number" vs "account number").
+ quoted_folds = {
+ fold for fold in candidates_by_fold if any(fold in quote or quote in fold for quote in answer_quotes)
+ }
+ unanswered = sorted(set(candidates_by_fold) - refuted_folds - quoted_folds)
+ if unanswered:
+ # Reported, not refused: refusing costs the whole slot contract and every criterion in it,
+ # where the only thing proven is that the producer stayed silent on a detected field.
+ LOG.info("copilot_request_slot_candidates_unanswered", unanswered=unanswered)
+
digest = request_slot_request_digest(request)
canonical_slots: list[CanonicalRequestSlotV1] = []
seen_paths: set[str] = set()
@@ -570,6 +624,7 @@ def canonicalize_request_slots(
count=len(canonical_slots),
datum_bindings=tuple(canonical_bindings),
datum_declines=tuple(canonical_declines),
+ candidate_refutations=envelope.candidate_refutations,
)
@@ -591,6 +646,11 @@ def _render_prompt(request: RequestSlotProducerInputV1) -> str:
ensure_ascii=True,
separators=(",", ":"),
),
+ detected_output_candidates=json.dumps(
+ list(request.detected_output_candidates),
+ ensure_ascii=True,
+ separators=(",", ":"),
+ ),
)
@@ -606,31 +666,57 @@ def _parse_envelope(raw: object) -> RequestSlotEnvelopeV1 | None:
return None
+# Fields the contract carries for the record rather than to be acted on. Two attempts that named
+# the same slots and bindings describe one request, so letting a differing rationale list refuse the
+# whole contract left a turn with no requested output at all (SKY-13226).
+_INFORMATIONAL_CONTRACT_FIELDS: frozenset[str] = frozenset({"candidate_refutations"})
+
+
+def request_slot_contract_disagreements(
+ first: RequestSlotContractV1,
+ second: RequestSlotContractV1,
+) -> tuple[str, ...]:
+ """Dotted paths of the fields two classifier attempts disagreed on — names only, never values.
+
+ A run that degrades on oscillation records that it happened but not what moved, which leaves the
+ difference between a producer flipping one judgment and one drifting wholesale unmeasurable.
+ """
+ fields: list[str] = []
+ for name in ("version", "request_digest", "count"):
+ if getattr(first, name) != getattr(second, name):
+ fields.append(name)
+ if len(first.slots) != len(second.slots):
+ fields.append("slots.count")
+ else:
+ for index, (first_slot, second_slot) in enumerate(zip(first.slots, second.slots, strict=True)):
+ for name in ("ordinal", "source_id", "plane", "pinability", "antecedent_family"):
+ if getattr(first_slot, name) != getattr(second_slot, name):
+ fields.append(f"slots.{index}.{name}")
+ if not (
+ first_slot.source_start < second_slot.source_end and second_slot.source_start < first_slot.source_end
+ ):
+ fields.append(f"slots.{index}.source_span")
+ if len(first.datum_bindings) != len(second.datum_bindings):
+ fields.append("datum_bindings.count")
+ else:
+ for index, (first_binding, second_binding) in enumerate(
+ zip(first.datum_bindings, second.datum_bindings, strict=True)
+ ):
+ for name in ("criterion_index", "datum_field", "datum_value", "criterion_outcome_sha256", "slot_id"):
+ if getattr(first_binding, name) != getattr(second_binding, name):
+ fields.append(f"datum_bindings.{index}.{name}")
+ if first.datum_declines != second.datum_declines:
+ fields.append("datum_declines")
+ if first.candidate_refutations != second.candidate_refutations:
+ fields.append("candidate_refutations")
+ return tuple(fields)
+
+
def request_slot_contracts_agree(
first: RequestSlotContractV1,
second: RequestSlotContractV1,
) -> bool:
- if first.version != second.version or first.request_digest != second.request_digest or first.count != second.count:
- return False
- slots_agree = all(
- first_slot.ordinal == second_slot.ordinal
- and first_slot.source_id == second_slot.source_id
- and first_slot.plane == second_slot.plane
- and first_slot.pinability == second_slot.pinability
- and first_slot.antecedent_family == second_slot.antecedent_family
- and first_slot.source_start < second_slot.source_end
- and second_slot.source_start < first_slot.source_end
- for first_slot, second_slot in zip(first.slots, second.slots, strict=True)
- )
- bindings_agree = len(first.datum_bindings) == len(second.datum_bindings) and all(
- first_binding.criterion_index == second_binding.criterion_index
- and first_binding.datum_field == second_binding.datum_field
- and first_binding.datum_value == second_binding.datum_value
- and first_binding.criterion_outcome_sha256 == second_binding.criterion_outcome_sha256
- and first_binding.slot_id == second_binding.slot_id
- for first_binding, second_binding in zip(first.datum_bindings, second.datum_bindings, strict=True)
- )
- return slots_agree and bindings_agree and first.datum_declines == second.datum_declines
+ return not request_slot_contract_disagreements(first, second)
async def produce_request_slots(
@@ -687,8 +773,18 @@ async def produce_request_slots(
except ValueError:
last_failure = RequestSlotProducerFailureKind.INVALID_OUTPUT
continue
- if candidate_contract is not None and request_slot_contracts_agree(candidate_contract, contract):
- return RequestSlotProducerResult.success(candidate_contract, attempts=attempt)
+ if candidate_contract is not None:
+ disagreements = request_slot_contract_disagreements(candidate_contract, contract)
+ decisive = tuple(field for field in disagreements if field not in _INFORMATIONAL_CONTRACT_FIELDS)
+ if not decisive:
+ if disagreements:
+ LOG.info(
+ "request-slot classifier agreed despite informational differences",
+ attempt=attempt,
+ fields=disagreements,
+ )
+ return RequestSlotProducerResult.success(candidate_contract, attempts=attempt)
+ LOG.info("request-slot classifier attempts disagreed", attempt=attempt, fields=disagreements)
candidate_contract = contract
if candidate_contract is not None:
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index e7389009a..b03a9f366 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -58,7 +58,10 @@ if TYPE_CHECKING:
from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult
from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext
from skyvern.forge.sdk.copilot.mcp_adapter import SkyvernOverlayMCPServer
- from skyvern.forge.sdk.copilot.output_extraction_plan import FrozenRequestedOutputExtractionCandidate
+ from skyvern.forge.sdk.copilot.output_extraction_plan import (
+ FrozenRequestedOutputExtractionCandidate,
+ RequestedOutputExtractionPlan,
+ )
from skyvern.forge.sdk.copilot.reached_download_target import ReachedDownloadTarget
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
from skyvern.forge.sdk.copilot.result_evidence import LoadedResultCompositionEvidence, ScoutObservationContract
@@ -221,7 +224,14 @@ class ScoutedInteraction(TypedDict):
# value answers. Recorded so authoring quotes a proven read instead of guessing a selector.
read_expression: NotRequired[str]
read_output_path: NotRequired[str]
+ # Whether the reader named this path or it was the only one left. A witness binds a value to a
+ # path, so a read that merely inherited the path by elimination is not evidence of that path.
+ read_output_path_source: NotRequired[str]
read_result_shape: NotRequired[str]
+ # The scalar the read actually returned, so a later binding can locate the element that still
+ # carries it rather than re-deriving one from labels. Bounded and scalar-only; turn-ephemeral,
+ # like raw_typed_value, and excluded from every persistence path.
+ read_result_value: NotRequired[str]
# Set when a live scout-time count()==1 probe found the captured selector matching >1 element on its
# source page; synthesis re-anchors or drops it rather than emitting a selector that strict-mode-fails.
ambiguous: NotRequired[bool]
@@ -258,6 +268,10 @@ class AgentContext:
injected_browser_state: BrowserState | None = None
heal_workflow_run_id: str | None = None
turn_intent: TurnIntent | None = None
+ # The streaming adapter narrates any context it is handed, so the design-phase latches live here
+ # rather than on the copilot subclass it is annotated for.
+ design_start_emitted: bool = False
+ design_end_emitted: bool = False
# Ephemeral carrier for SDK-action run reuse, bounded by browser sessions touched in one Copilot run.
sdk_action_workflow_run_ids_by_browser_session: dict[SdkActionWorkflowRunCacheKey, str] = field(
default_factory=dict
@@ -479,10 +493,16 @@ class AgentContext:
synthesized_block_offered: bool = False
synthesized_block_offered_trajectory_len: int = 0
synthesized_block_offered_goal_complete: bool = False
+ # Probe-validated value elements the model designated, each pinned to the page it was resolved on.
+ requested_output_designations: list[dict[str, Any]] = field(default_factory=list)
requested_output_extraction_candidate: FrozenRequestedOutputExtractionCandidate | None = None
# Candidate frozen by an imposition that has not been persisted yet; promoted to the committed
# candidate only once the update it rode in on succeeds.
pending_requested_output_extraction_candidate: FrozenRequestedOutputExtractionCandidate | None = None
+ # The last plan that bound every requested path. Derivation reads the freshest packet, which is
+ # usually truncated or unbindable, so a plan that did bind is kept rather than re-derived away
+ # before the imposition that needs it.
+ last_bound_requested_output_extraction_plan: RequestedOutputExtractionPlan | None = None
# Set by the imposition seam when a goal-complete spine is on its way into a draft; the successful update
# promotes it to the landed latch only when the persisted draft covers the freshly scouted spine.
pending_goal_complete_landing: bool = False
@@ -545,6 +565,9 @@ class AgentContext:
# Expression of an in-flight evaluate, stashed pre-dispatch: the MCP response carries only the
# result, so a post-hook that wants the expression must receive it from the invocation side.
pending_scout_read_expression: str | None = None
+ # Requested output the in-flight evaluate says it fills. Without it a read can only be attributed
+ # when the turn requests exactly one output, so a multi-field request binds nothing.
+ pending_scout_read_output_path: str | None = None
# Connected overlay used by bounded pre-click evidence probes; declared so capture code accesses it
# directly instead of silently accepting a dynamically attached dependency.
discovery_mcp_server: SkyvernOverlayMCPServer | None = None
diff --git a/skyvern/forge/sdk/copilot/session_factory.py b/skyvern/forge/sdk/copilot/session_factory.py
index ba7408261..f6dfbf380 100644
--- a/skyvern/forge/sdk/copilot/session_factory.py
+++ b/skyvern/forge/sdk/copilot/session_factory.py
@@ -2,13 +2,20 @@
from __future__ import annotations
+import json
+import os
from collections.abc import Callable
+from itertools import count
+from pathlib import Path
from typing import Any
import structlog
from agents.memory.sqlite_session import SQLiteSession
from agents.run_config import CallModelData, ModelInputData
+from agents.run_context import RunContextWrapper
+from pydantic import BaseModel
+from skyvern.config import settings
from skyvern.forge.sdk.agents.context import (
compact_agent_messages_for_llm,
get_agent_message_field,
@@ -108,6 +115,54 @@ def copilot_call_model_input_filter(data: CallModelData[Any]) -> ModelInputData:
def _copilot_call_model_input_filter(data: CallModelData[Any], *, token_budget: int) -> ModelInputData:
+ model_data = _filter_to_budget(data, token_budget=token_budget)
+ _maybe_dump_model_input(data, model_data)
+ return model_data
+
+
+_MODEL_CALL_SEQ = count()
+
+
+def _jsonable(item: Any) -> Any:
+ if isinstance(item, BaseModel):
+ return item.model_dump(mode="json")
+ return item
+
+
+def _maybe_dump_model_input(data: CallModelData[Any], model_data: ModelInputData) -> None:
+ """Record the exact model input so a prompt or tool-schema change can be replayed offline.
+
+ Written here rather than at the Runner call because this is the only point that sees what the
+ model actually receives — after session merge, compaction, and every budget layer above. That
+ includes page text and tool results, so writing takes both an explicit path and a local
+ environment rather than the path alone.
+ """
+ dump_dir = os.getenv("COPILOT_DUMP_MODEL_INPUTS")
+ if not dump_dir or settings.ENV != "local":
+ return
+ try:
+ from skyvern.forge.sdk.copilot.enforcement import requested_output_paths_for_derivation
+
+ # CallModelData carries the run context itself; only some entry points hand over a wrapper.
+ ctx = data.context.context if isinstance(data.context, RunContextWrapper) else data.context
+ try:
+ requested_output_paths = sorted(requested_output_paths_for_derivation(ctx)) if ctx else []
+ except Exception:
+ requested_output_paths = []
+ target = Path(dump_dir)
+ target.mkdir(parents=True, exist_ok=True)
+ payload = {
+ "instructions": model_data.instructions,
+ "input": [_jsonable(item) for item in model_data.input],
+ "requested_output_paths": requested_output_paths,
+ }
+ path = target / f"call-{next(_MODEL_CALL_SEQ):04d}.json"
+ path.write_text(json.dumps(payload, indent=2, default=str))
+ except Exception:
+ LOG.warning("Failed to dump copilot model input", exc_info=True)
+
+
+def _filter_to_budget(data: CallModelData[Any], *, token_budget: int) -> ModelInputData:
"""Token-budget enforcement applied just before each model call.
Graduated pruning:
diff --git a/skyvern/forge/sdk/copilot/tools/__init__.py b/skyvern/forge/sdk/copilot/tools/__init__.py
index 399e61e30..caf035aad 100644
--- a/skyvern/forge/sdk/copilot/tools/__init__.py
+++ b/skyvern/forge/sdk/copilot/tools/__init__.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+import asyncio
import json
import time
from typing import Any
@@ -9,6 +10,7 @@ from typing import Any
import structlog
from agents import function_tool
from agents.run_context import RunContextWrapper
+from typing_extensions import TypedDict
from skyvern.forge import app as app
from skyvern.forge.sdk.copilot.build_phase import (
@@ -24,12 +26,14 @@ from skyvern.forge.sdk.copilot.composition_evidence import (
from skyvern.forge.sdk.copilot.composition_evidence import workflow_target_url as workflow_target_url
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.loop_detection import record_tool_step_result_for_ctx
+from skyvern.forge.sdk.copilot.output_extraction_plan import value_designation_probe_expression
from skyvern.forge.sdk.copilot.output_utils import (
_INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY as _INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY,
)
from skyvern.forge.sdk.copilot.output_utils import (
sanitize_tool_result_for_llm,
)
+from skyvern.forge.sdk.copilot.runtime import AgentContext
from skyvern.forge.sdk.copilot.screenshot_utils import enqueue_screenshot_from_result
from skyvern.forge.sdk.copilot.secret_scrub import scrub_secrets_from_structure
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
@@ -39,6 +43,7 @@ from skyvern.forge.sdk.copilot.workflow_yaml import apply_block_edit, delete_blo
from skyvern.utils.yaml_loader import safe_load_no_dates
from ._shared import _COMPOSITION_STRIPPED_HTML_MAX_CHARS as _COMPOSITION_STRIPPED_HTML_MAX_CHARS
+from ._shared import _DISCOVERY_PER_CALL_TIMEOUT_SECONDS as _DISCOVERY_PER_CALL_TIMEOUT_SECONDS
from ._shared import _FAILED_BLOCK_STATUSES as _FAILED_BLOCK_STATUSES
from ._shared import BLOCK_RUNNING_TOOLS as BLOCK_RUNNING_TOOLS
from ._shared import COPILOT_FINAL_REPLY_RESERVE_SECONDS as COPILOT_FINAL_REPLY_RESERVE_SECONDS
@@ -462,19 +467,122 @@ async def delete_block_tool(ctx: RunContextWrapper, label: str) -> str:
return await _persist_block_scoped_edit(copilot_ctx, "delete_block", workflow_yaml, arguments)
+_MAX_REQUESTED_OUTPUT_READS = 8
+
+# Probe outcomes that located the designated value without resolving it to a single element.
+_DESIGNATION_ERRORS_THAT_SAW_THE_VALUE = frozenset({"text-ambiguous", "path-unstable"})
+
+
+class RequestedOutputRead(TypedDict, total=False):
+ output_path: str
+ value_text: str
+ label: str
+
+
+async def _validate_requested_output_reads(
+ copilot_ctx: AgentContext, reads: list[RequestedOutputRead]
+) -> tuple[list[dict[str, Any]], list[dict[str, str]]]:
+ """Resolve each designated value against the live page; keep what pins, report what does not."""
+ validated: list[dict[str, Any]] = []
+ rejected: list[dict[str, str]] = []
+ server = copilot_ctx.discovery_mcp_server
+ if len(reads) > _MAX_REQUESTED_OUTPUT_READS:
+ rejected.append({"output_path": "", "error": f"only the first {_MAX_REQUESTED_OUTPUT_READS} reads were probed"})
+ for read in reads[:_MAX_REQUESTED_OUTPUT_READS]:
+ raw_path = str(read.get("output_path") or "")
+ value_text = str(read.get("value_text") or "").strip()
+ label = str(read.get("label") or "").strip()
+ # The model designates by the output's own name; "output." is our internal path prefix.
+ output_path = raw_path if raw_path.startswith("output.") else f"output.{raw_path}"
+ if not raw_path or not value_text:
+ rejected.append({"output_path": raw_path, "error": "malformed"})
+ continue
+ if server is None:
+ rejected.append({"output_path": output_path, "error": "no-browser"})
+ continue
+ expression = value_designation_probe_expression(value_text, label)
+ try:
+ raw = await asyncio.wait_for(
+ server.call_internal_tool("skyvern_evaluate", {"expression": expression}),
+ timeout=_DISCOVERY_PER_CALL_TIMEOUT_SECONDS,
+ )
+ except Exception:
+ rejected.append({"output_path": output_path, "error": "probe-failed"})
+ continue
+ payload = (raw.get("data") or {}).get("result") if isinstance(raw, dict) and raw.get("ok") else None
+ if isinstance(payload, str):
+ try:
+ payload = json.loads(payload)
+ except (TypeError, ValueError):
+ payload = None
+ if not isinstance(payload, dict) or payload.get("error") or not isinstance(payload.get("text"), str):
+ error = str(payload.get("error")) if isinstance(payload, dict) and payload.get("error") else "no-result"
+ detail = {"output_path": output_path, "error": error, "value_text": value_text[:40]}
+ if isinstance(payload, dict):
+ for field_name in ("match_count", "visible_count"):
+ if payload.get(field_name) is not None:
+ detail[field_name] = str(payload[field_name])
+ rejected.append(detail)
+ # The probe failed to pin one element, not to find the value: several leaves showed it,
+ # or no stable path reached the one that did. The value is still what the model read off
+ # the page, and the relation capture carries its own uniqueness proof for it.
+ if error in _DESIGNATION_ERRORS_THAT_SAW_THE_VALUE and isinstance(payload, dict):
+ validated.append(
+ {
+ "output_path": output_path,
+ "selector": "",
+ "text": str(payload.get("text") or value_text),
+ "url": str(payload.get("url") or ""),
+ }
+ )
+ continue
+ resolved_selector = payload.get("selector")
+ validated.append(
+ {
+ "output_path": output_path,
+ "selector": resolved_selector if isinstance(resolved_selector, str) else "",
+ "match_count": payload.get("match_count"),
+ "position": payload.get("position"),
+ "text": payload["text"],
+ "url": payload.get("url") or "",
+ }
+ )
+ return validated, rejected
+
+
@function_tool(name_override="synthesize_demonstrated_block")
-async def synthesize_demonstrated_block_tool(ctx: RunContextWrapper) -> str:
+async def synthesize_demonstrated_block_tool(
+ ctx: RunContextWrapper,
+ requested_output_reads: list[RequestedOutputRead] | None = None,
+) -> str:
"""Return code-block source built from the steps you already performed in the browser.
This is compiled from your recorded interactions, so it reflects what actually happened rather
than what you remember doing. Use it as the body of a code block instead of writing the steps
again from memory, and edit it if the block needs to do more.
+
+ When the request asks the workflow to return values you can see on the page, pass
+ requested_output_reads: for each requested output, `value_text` is the value exactly as it is
+ rendered ("8.9K", not 8900) and `label` is the heading it sits under. The page resolves those to
+ the element and compiles a pinned deterministic read; you do not need to know its markup.
"""
copilot_ctx = ctx.context
- arguments: dict[str, Any] = {}
+ arguments: dict[str, Any] = {"requested_output_reads": requested_output_reads or []}
loop_error = _tool_loop_error(copilot_ctx, "synthesize_demonstrated_block", arguments)
if loop_error:
return json.dumps({"ok": False, "error": loop_error})
+ rejected: list[dict[str, str]] = []
+ if requested_output_reads:
+ validated, rejected = await _validate_requested_output_reads(copilot_ctx, requested_output_reads)
+ copilot_ctx.requested_output_designations = validated
+ LOG.info(
+ "copilot_requested_output_designations",
+ pinned=[{"output_path": d["output_path"], "text": d["text"][:40]} for d in validated if d["selector"]],
+ value_only=[
+ {"output_path": d["output_path"], "text": d["text"][:40]} for d in validated if not d["selector"]
+ ],
+ rejected=rejected,
+ )
source = _demonstrated_step_sources(copilot_ctx)
if not source:
result = {
@@ -482,7 +590,10 @@ async def synthesize_demonstrated_block_tool(ctx: RunContextWrapper) -> str:
"error": "Nothing has been demonstrated in the browser yet, so there are no steps to compile.",
}
else:
- result = {"ok": True, "data": {"code": source}}
+ data: dict[str, Any] = {"code": source}
+ if rejected:
+ data["rejected_output_reads"] = rejected
+ result = {"ok": True, "data": data}
record_tool_step_result_for_ctx(copilot_ctx, "synthesize_demonstrated_block", arguments, result)
return json.dumps(result)
diff --git a/skyvern/forge/sdk/copilot/tools/_shared.py b/skyvern/forge/sdk/copilot/tools/_shared.py
index a8f722ed2..10679b9ae 100644
--- a/skyvern/forge/sdk/copilot/tools/_shared.py
+++ b/skyvern/forge/sdk/copilot/tools/_shared.py
@@ -17,14 +17,23 @@ from skyvern.forge.sdk.copilot.composition_browser_expressions import (
COMPOSITION_STRIPPED_HTML_MAX_CHARS as _COMPOSITION_STRIPPED_HTML_MAX_CHARS,
)
from skyvern.forge.sdk.copilot.composition_browser_expressions import (
- COMPOSITION_STRUCTURED_EVIDENCE_EXPRESSION as _COMPOSITION_STRUCTURED_EVIDENCE_EXPRESSION,
+ COMPOSITION_STRUCTURED_EVIDENCE_MAX_CHARS as _COMPOSITION_STRUCTURED_EVIDENCE_MAX_CHARS,
)
from skyvern.forge.sdk.copilot.composition_browser_expressions import (
- COMPOSITION_STRUCTURED_EVIDENCE_MAX_CHARS as _COMPOSITION_STRUCTURED_EVIDENCE_MAX_CHARS,
+ composition_structured_evidence_expression,
+)
+from skyvern.forge.sdk.copilot.composition_evidence import (
+ clearable_dismiss_texts,
+ has_bounded_page_schema,
+ packet_describes_a_clearable_overlay,
+ parse_composition_structured,
)
-from skyvern.forge.sdk.copilot.composition_evidence import has_bounded_page_schema, parse_composition_structured
from skyvern.forge.sdk.copilot.context import CopilotContext
-from skyvern.forge.sdk.copilot.enforcement import TOTAL_TIMEOUT_SECONDS, _elapsed_run_seconds
+from skyvern.forge.sdk.copilot.enforcement import (
+ TOTAL_TIMEOUT_SECONDS,
+ _elapsed_run_seconds,
+ _requested_output_labels_by_path,
+)
from skyvern.forge.sdk.copilot.runtime import AgentContext, resolve_browser_state_for_context
from skyvern.forge.sdk.copilot.task_output_envelope import (
_TASK_ENVELOPE_BLOCK_TYPES,
@@ -645,6 +654,8 @@ def _append_flow_evidence(copilot_ctx: Any, evidence: dict[str, Any], *, reached
"evidence": evidence,
"reached_via": reached_via,
"had_bounded_schema": has_bounded_page_schema(evidence),
+ "obstructed": packet_describes_a_clearable_overlay(evidence),
+ "dismiss_texts": sorted(clearable_dismiss_texts(evidence)),
"step": step,
}
)
@@ -712,6 +723,19 @@ async def _composition_get_html(copilot_ctx: Any, *, skip_raw: bool = False) ->
return "", str(error) if error else None, False, True
+def _requested_capture_targets(copilot_ctx: object) -> tuple[str, ...]:
+ """The labels this turn asked for, so capture resolves them rather than guessing which relations matter."""
+ if not isinstance(copilot_ctx, AgentContext):
+ return ()
+ targets: list[str] = []
+ for labels in _requested_output_labels_by_path(copilot_ctx).values():
+ for label in labels:
+ text = label.strip()
+ if text and text not in targets:
+ targets.append(text)
+ return tuple(targets)
+
+
async def _composition_get_structured_evidence(
copilot_ctx: Any,
*,
@@ -727,7 +751,8 @@ async def _composition_get_structured_evidence(
try:
result = await asyncio.wait_for(
server.call_internal_tool(
- "skyvern_evaluate", {"expression": _COMPOSITION_STRUCTURED_EVIDENCE_EXPRESSION}
+ "skyvern_evaluate",
+ {"expression": composition_structured_evidence_expression(_requested_capture_targets(copilot_ctx))},
),
timeout=timeout_seconds,
)
diff --git a/skyvern/forge/sdk/copilot/tools/composition_capture.py b/skyvern/forge/sdk/copilot/tools/composition_capture.py
index 84707ae98..fc17e04e4 100644
--- a/skyvern/forge/sdk/copilot/tools/composition_capture.py
+++ b/skyvern/forge/sdk/copilot/tools/composition_capture.py
@@ -38,6 +38,7 @@ from ._shared import (
_discovery_extract_current_url,
_discovery_navigate,
_fallback_page_info,
+ _requested_capture_targets,
_same_page_ignoring_fragment,
_workflow_verification_evidence,
)
@@ -270,7 +271,12 @@ async def _composition_evidence_after_navigation_failure(
# error still parses via the stripped-body evaluate instead of yielding hollow evidence.
html, html_error, html_truncated, _ = await _composition_get_html(ctx)
if html_error is None:
- evidence = parse_composition_html(html, inspected_url=inspected_url, current_url=current_url)
+ evidence = parse_composition_html(
+ html,
+ inspected_url=inspected_url,
+ current_url=current_url,
+ requested_targets=_requested_capture_targets(ctx),
+ )
evidence = _composition_add_inspection_warning(
evidence,
f"navigation_error_before_html_capture: {navigation_error}",
@@ -476,7 +482,12 @@ async def _capture_composition_evidence(
# stripped path only so a slow page is still retried without re-serializing the full DOM.
if used_stripped:
skip_raw = True
- evidence = parse_composition_html(html, inspected_url=inspected_url, current_url=current_url)
+ evidence = parse_composition_html(
+ html,
+ inspected_url=inspected_url,
+ current_url=current_url,
+ requested_targets=_requested_capture_targets(copilot_ctx),
+ )
used_structured = False
if _composition_capture_settled(evidence):
break
diff --git a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
index 09076a7b1..dc8c52837 100644
--- a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
+++ b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
@@ -15,7 +15,6 @@ from skyvern.forge.sdk.copilot.build_phase import (
advance_to_composing,
)
from skyvern.forge.sdk.copilot.code_block_synthesis import synthesize_code_block
-from skyvern.forge.sdk.copilot.completion_criteria_store import requested_output_paths
from skyvern.forge.sdk.copilot.composition_browser_expressions import scout_control_state_expression
from skyvern.forge.sdk.copilot.config import (
BlockAuthoringPolicy,
@@ -23,10 +22,12 @@ from skyvern.forge.sdk.copilot.config import (
)
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.credential_resolution import load_credentials
+from skyvern.forge.sdk.copilot.enforcement import requested_output_paths_for_derivation
from skyvern.forge.sdk.copilot.mcp_adapter import SchemaOverlay
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy, resolve_credential_for_live_page
from skyvern.forge.sdk.copilot.request_slots import is_canonical_request_slot_path
from skyvern.forge.sdk.copilot.runtime import AgentContext, ScoutedInteraction
+from skyvern.forge.sdk.copilot.secret_redaction import redact_raw_secrets_for_prompt
from skyvern.forge.sdk.copilot.secret_scrub import registered_scrub_values
from skyvern.forge.sdk.copilot.typed_value_policy import safe_typed_default_value, should_reject_type_text_value
from skyvern.forge.sdk.schemas.credentials import Credential
@@ -277,6 +278,7 @@ async def _evaluate_pre_hook(
# Cleared up front so an early reject cannot leave a prior evaluate's expression for this
# call's post-hook to consume.
ctx.pending_scout_read_expression = None
+ ctx.pending_scout_read_output_path = None
raw_expression = params.get("expression")
expr = raw_expression.lower() if isinstance(raw_expression, str) else ""
if ".click()" in expr or ".click(" in expr:
@@ -286,6 +288,9 @@ async def _evaluate_pre_hook(
}
if isinstance(raw_expression, str) and raw_expression.strip():
ctx.pending_scout_read_expression = raw_expression
+ raw_output_path = params.get("output_path")
+ if isinstance(raw_output_path, str) and raw_output_path.strip():
+ ctx.pending_scout_read_output_path = raw_output_path.strip()
return None
@@ -937,7 +942,57 @@ async def _type_text_post_hook(
return result
-def _record_scouted_read(ctx: AgentContext, *, expression: str, data: dict[str, Any], url: str) -> None:
+# composition_evidence caps a captured value_text at 240 chars, so a witness longer than that cannot
+# match one and is only page text riding along in the trajectory.
+_WITNESSED_READ_VALUE_MAX_CHARS = 240
+
+
+def _sole_scalar_leaf(result: object) -> object | None:
+ """The one scalar anywhere inside a result, or None if it holds none or more than one.
+
+ A read that answers with a value often returns it wrapped — a one-key dict, a one-element list —
+ and the wrapper says nothing about which element on the page carries it. More than one scalar
+ means the read described a page instead of answering with a value, so the whole structure is
+ walked; stopping early would call a shallow scalar sole while a deeper one went unseen.
+ """
+ found: object | None = None
+ pending: list[object] = [result]
+ while pending:
+ current = pending.pop()
+ if isinstance(current, (str, int, float)):
+ if found is not None:
+ return None
+ found = current
+ elif isinstance(current, dict):
+ pending.extend(current.values())
+ elif isinstance(current, (list, tuple, set)):
+ pending.extend(current)
+ return found
+
+
+def _witnessed_scalar_value(result: object) -> str:
+ """The scalar a read returned, so a later binding can find what still carries it."""
+ scalar = _sole_scalar_leaf(result)
+ if scalar is None or isinstance(scalar, bool):
+ return ""
+ text = str(scalar).strip()
+ if not text or len(text) > _WITNESSED_READ_VALUE_MAX_CHARS:
+ return ""
+ # Redacted text cannot support the exact match this exists for, so a value redaction would alter
+ # is dropped rather than kept in a form nothing can bind.
+ if redact_raw_secrets_for_prompt(text) != text:
+ return ""
+ return text
+
+
+def _record_scouted_read(
+ ctx: AgentContext,
+ *,
+ expression: str,
+ data: dict[str, Any],
+ url: str,
+ declared_output_path: str | None = None,
+) -> None:
"""Keep a read the scout proved on the live page so authoring replays it instead of guessing a
locator for a value it has already seen. A structured result is kept as readily as a scalar: the
shape is what a later binding needs. Naming never gates keeping the evidence."""
@@ -947,27 +1002,76 @@ def _record_scouted_read(ctx: AgentContext, *, expression: str, data: dict[str,
if isinstance(result, (list, dict)) and not result:
return
output_path = "output.scouted_read"
+ witnessed = _witnessed_scalar_value(result)
policy = ctx.request_policy
+ requested: set[str] = set()
if isinstance(policy, RequestPolicy):
- requested = requested_output_paths(policy.graded_completion_criteria())
+ # Derivation's set, not the graded criteria projected to output_path: rekeying clears that
+ # field while the criterion stays graded, so a witnessed value stayed anonymous and
+ # completion verification had nothing for an outcome the scout had already seen.
+ requested = requested_output_paths_for_derivation(ctx)
named = sorted(path for path in requested if not is_canonical_request_slot_path(path))
# A rekeyed requested output carries a digest instead of a word, which still keys the
# producer to the criterion it has to satisfy. Reading into an anonymous path instead leaves
# completion verification with no evidence for an outcome the scout already demonstrated.
candidates = named or sorted(requested)
- if len(candidates) == 1:
+ # Owning the requested output takes a value the read actually observed, not merely being the
+ # only path on offer: a read that came back holding no single value describes the page, and
+ # promoting it leaves the run claiming an answer nothing witnessed.
+ if len(candidates) == 1 and witnessed:
output_path = candidates[0]
+ # Counting candidates can only attribute a read when the turn requests a single output; a request
+ # for several fields needs the reader to say which one it just read.
+ if declared_output_path and (
+ declared_output_path in requested or (not requested and declared_output_path.startswith("output."))
+ ):
+ output_path = declared_output_path
+ # The same tool inspects a page and reads a value, so attribution by elimination promotes an
+ # inspection to the requested output. Report what owning the output by declaration alone would
+ # bind, so the cost of that rule is known before it decides anything.
+ LOG.info(
+ "copilot_scouted_read_attribution",
+ declared=bool(declared_output_path),
+ attributed_by="declaration" if declared_output_path == output_path else "elimination",
+ bound_output_path=output_path,
+ declaration_only_output_path=declared_output_path or "output.scouted_read",
+ requested_output_count=len(requested),
+ read_result_shape=type(result).__name__,
+ )
interaction: ScoutedInteraction = {
"tool_name": "read_value",
"read_expression": expression,
"read_output_path": output_path,
+ "read_output_path_source": "declared" if declared_output_path == output_path else "elimination",
"read_result_shape": type(result).__name__,
}
+ if witnessed:
+ interaction["read_result_value"] = witnessed
if url:
interaction["source_url"] = url
ctx.scout_trajectory.append(interaction)
+def _unread_requested_output_paths(ctx: AgentContext) -> list[str]:
+ """Requested outputs no read has yet answered with a value.
+
+ A read that only gathered candidates still names the path it was aiming at, so counting every
+ read as a claim made probing indistinguishable from progress: the turn could inspect a tile
+ repeatedly and reach authoring with nothing bound to the output it was asked for.
+ """
+ # The same set derivation binds and the offer names: rekeying clears a criterion's output_path while it stays
+ # graded, so projecting the graded criteria alone loses the path the binder still owes (SKY-13226).
+ requested = requested_output_paths_for_derivation(ctx)
+ if not requested:
+ return []
+ claimed = {
+ str(interaction.get("read_output_path") or "")
+ for interaction in ctx.scout_trajectory
+ if interaction.get("tool_name") == "read_value" and interaction.get("read_result_value") is not None
+ }
+ return sorted(requested - claimed)
+
+
async def _evaluate_post_hook(
result: dict[str, Any],
raw: dict[str, Any],
@@ -1003,15 +1107,50 @@ async def _evaluate_post_hook(
# The MCP response never echoes the expression; the pre-hook stashed it from the invocation.
read_expression = ctx.pending_scout_read_expression or ""
ctx.pending_scout_read_expression = None
+ declared_output_path = ctx.pending_scout_read_output_path
+ ctx.pending_scout_read_output_path = None
trajectory_len_before = len(ctx.scout_trajectory)
- _record_scouted_read(ctx, expression=read_expression, data=data, url=url)
+ _record_scouted_read(
+ ctx,
+ expression=read_expression,
+ data=data,
+ url=url,
+ declared_output_path=declared_output_path,
+ )
if len(ctx.scout_trajectory) > trajectory_len_before:
+ recorded = ctx.scout_trajectory[-1]
LOG.info(
"copilot_scouted_read_recorded",
- read_result_shape=ctx.scout_trajectory[-1].get("read_result_shape"),
- read_output_path=ctx.scout_trajectory[-1].get("read_output_path"),
+ read_result_shape=recorded.get("read_result_shape"),
+ read_output_path=recorded.get("read_output_path"),
+ witnessed_value_kept=bool(recorded.get("read_result_value")),
trajectory_len=len(ctx.scout_trajectory),
)
+ claimed_path = str(recorded.get("read_output_path") or "")
+ if claimed_path.startswith("output.") and not recorded.get("read_result_value"):
+ # The read owns an output the run has to register, but returned a shape holding no single
+ # value, so nothing downstream can say which element on the page carries the answer. Said
+ # as a bare fact it went unacted on six times in one turn while the sibling signal below
+ # carried a hint and was followed, so it names the next call too (SKY-13226).
+ data["claimed_output_without_a_single_value"] = claimed_path
+ data["claimed_output_read_hint"] = (
+ f"That expression gathered candidates rather than evaluating to one value, so nothing "
+ f"records what {claimed_path} is. Read the single value on its own — an expression whose "
+ f"result is just that value — and pass output_path={claimed_path} on that call."
+ )
+ LOG.info(
+ "copilot_scouted_read_claimed_output_without_value",
+ read_output_path=claimed_path,
+ read_result_shape=recorded.get("read_result_shape"),
+ )
+ unread_requested = _unread_requested_output_paths(ctx)
+ if unread_requested:
+ data["requested_outputs_still_unread"] = unread_requested
+ data["requested_output_read_hint"] = (
+ "Read each of these values and pass output_path=<the path> on that evaluate call. A read "
+ "that names any other path is kept as exploration and registers no output for the run."
+ )
+ LOG.info("copilot_requested_output_unread_after_read", unread=unread_requested)
if _copilot_block_authoring_policy(
ctx
) == BlockAuthoringPolicy.CODE_ONLY_BROWSER and _code_only_has_target_page_evidence(data):
@@ -1213,6 +1352,20 @@ def _build_skyvern_mcp_overlays(
"evaluate": SchemaOverlay(
description=_evaluate_overlay_description(block_authoring_policy),
hide_params=frozenset({"session_id", "cdp_url"}),
+ copilot_params={
+ "output_path": {
+ "type": "string",
+ "description": (
+ "The requested output this read fills, such as 'output.visitors'. Set it "
+ "whenever the expression reads a value the user asked for, so the workflow "
+ "returns that value under that name. Read each requested value in its own "
+ "call rather than one expression returning several. Naming a path says the "
+ "expression evaluates to that one value, which is what lets the workflow "
+ "find it again; an expression that gathers candidates is exploration, so "
+ "leave the path off and name it on the follow-up read of the value itself."
+ ),
+ }
+ },
requires_browser=True,
timeout=30,
pre_hook=_evaluate_pre_hook,
diff --git a/skyvern/forge/sdk/copilot/tools/scouting.py b/skyvern/forge/sdk/copilot/tools/scouting.py
index a1e18a398..946e302e9 100644
--- a/skyvern/forge/sdk/copilot/tools/scouting.py
+++ b/skyvern/forge/sdk/copilot/tools/scouting.py
@@ -55,6 +55,7 @@ from skyvern.forge.sdk.copilot.composition_evidence import (
has_actionable_steer_content,
has_bounded_page_schema,
has_witnessed_value_content,
+ packet_describes_a_clearable_overlay,
)
from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
from skyvern.forge.sdk.copilot.context import FillCarry
@@ -323,16 +324,27 @@ async def _capture_scout_role_name(ctx: AgentContext, selector: str | None) -> N
if not selector:
return
parsed = _role_name_from_selector(selector)
+ source = "selector"
if parsed is not None:
role, name = parsed
else:
+ source = "page_read"
captured = await _capture_accessible_role_name(
ctx, selector, timeout_seconds=_PRE_NAVIGATION_ROLE_NAME_TIMEOUT_SECONDS
)
if captured is None:
+ # A read that never answered and one that answered namelessly leave the same empty
+ # trajectory, and only the first is a timeout worth widening.
+ LOG.info("copilot_scout_role_name_unavailable", reason="page_read_failed", source=source)
return
role, name = captured
if not role or not name:
+ LOG.info(
+ "copilot_scout_role_name_unavailable",
+ reason="empty_role" if not role else "empty_name",
+ source=source,
+ role=role,
+ )
return
ctx.pending_scout_role_name = (selector, role, name)
@@ -1393,7 +1405,9 @@ _EVALUATE_ACTIONABLE_ACT_INSTRUCTION = (
)
_EVALUATE_RESULT_COMPOSITION_INSTRUCTION = (
"Loaded results are already visible on the current page; inspect this page for composition or author an "
- "extraction/validation block from the loaded results instead of re-reading it."
+ "extraction/validation block from the loaded results instead of re-reading it. For each requested output "
+ "value you can see, pass it to synthesize_demonstrated_block's requested_output_reads as the value exactly "
+ "as rendered plus the label above it, and the page will pin the read for you."
)
@@ -1792,7 +1806,21 @@ async def _maybe_steer_evaluate_to_action(
return False
record_scouted_output_coverage(ctx, parsed, contract=contract)
_record_scout_page_observation(ctx, parsed)
- loaded_results = _mint_current_loaded_result_source(ctx, parsed, url=url)
+ # An overlay the loop can clear takes precedence over reading what it obscured; the dismiss
+ # control is already among the actionable targets, so the model chooses rather than being
+ # vetoed, and the next observation sees the page. Checked before results are minted rather
+ # than after: a packet describing only a dialog has no results to mint, so gating this on
+ # having minted some left it dead in the one case it exists for.
+ overlay_only = packet_describes_a_clearable_overlay(parsed)
+ if overlay_only:
+ LOG.info("copilot_evaluate_result_composition_deferred_to_obstruction", url=url)
+ # Recording an observation is independent of whether it steers. A page whose relations do not
+ # mint a loaded result is steered as actionable instead, and appending only on the composition
+ # branch discarded those packets outright: the log page carrying the requested count was
+ # observed, steered as actionable, and never reached the binder at all (SKY-13226).
+ if has_bounded_page_schema(parsed):
+ _append_flow_evidence(ctx, parsed, reached_via="current_page")
+ loaded_results = None if overlay_only else _mint_current_loaded_result_source(ctx, parsed, url=url)
if loaded_results is not None:
_reset_evaluate_tracker(ctx)
ctx.latest_evaluate_result_composition_steer = loaded_results
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index 12bb6f2e9..b23425836 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -2750,6 +2750,23 @@ def _metadata_has_mapping_item(raw_metadata: object) -> bool:
return any(_raw_metadata_item_mapping(item) is not None for item in _code_artifact_metadata_items(raw_metadata))
+def _goal_paths_rooted_as_the_code_returns_them(required_paths: set[str], code: str) -> set[str]:
+ """Goal paths named the way ``code`` names them.
+
+ A goal path is written in the block's own namespace while a plan-derived extraction returns the
+ requested-output namespace, so one calls the value `visitors` and the other
+ `output.visitors`; compared raw, a value the code returns reads as absent. Re-rooting is applied
+ only where it is what the code actually returns, so a block returning a bare mapping keeps naming
+ its keys bare.
+ """
+ if not required_paths:
+ return required_paths
+ produced = _code_block_produced_output_paths(code)
+ return {
+ f"output.{path}" if path not in produced and f"output.{path}" in produced else path for path in required_paths
+ }
+
+
def _metadata_item_goal_value_paths(item: Mapping[str, Any] | None) -> set[str]:
if item is None:
return set()
@@ -5278,7 +5295,12 @@ def _replace_direct_child_local_return(
if len(returns) != 1:
return ""
return_node = returns[0]
- if not isinstance(return_node.value, ast.Name) or return_node.value.id != child_name:
+ # A single required path and a single returned local leave nothing to choose between, so the
+ # local carries the value whatever the author named it.
+ if not isinstance(return_node.value, ast.Name):
+ return ""
+ returned_name = return_node.value.id
+ if returned_name != child_name and returned_name in declaration_children:
return ""
if return_node.end_lineno is None:
return ""
@@ -5290,7 +5312,7 @@ def _replace_direct_child_local_return(
return ""
indent = indent_match.group(0)
pairs = ", ".join(
- f'"{name}": None' if name in declaration_children else f'"{name}": {name}' for name in child_names
+ f'"{name}": None' if name in declaration_children else f'"{name}": {returned_name}' for name in child_names
)
replacement = f'{indent}return {{"output": {{{pairs}}}}}'
return "\n".join(
@@ -6789,6 +6811,31 @@ def _is_allowed_browser_read_call(node: ast.Call, bindings: _BrowserBindings) ->
return False
+def _submission_returns_a_value(code: str) -> bool:
+ """Whether the submission hands a value back, read from its syntax rather than its text."""
+ try:
+ tree = ast.parse(textwrap.dedent(code or "").strip())
+ except SyntaxError:
+ return False
+ return any(isinstance(node, ast.Return) and node.value is not None for node in ast.walk(tree))
+
+
+def _submitted_code_fingerprint(code: str) -> dict[str, object]:
+ """Shape of what the author submitted, for attributing a lost extraction after imposition."""
+ stripped = textwrap.dedent(code or "").strip()
+ try:
+ tree: ast.Module | None = ast.parse(stripped)
+ except SyntaxError:
+ tree = None
+ return {
+ "submitted_sha256": hashlib.sha256(stripped.encode()).hexdigest()[:12],
+ "submitted_chars": len(stripped),
+ "submitted_parses": tree is not None,
+ "submitted_returns": tree is not None and any(isinstance(node, ast.Return) for node in ast.walk(tree)),
+ "submitted_reads_page": any(marker in stripped for marker in ("page.evaluate", "inner_text", "text_content")),
+ }
+
+
def _browser_surface_for_code(code: str) -> tuple[list[_BrowserMutationSignature], list[str], list[str]]:
tree = _wrapped_code_ast(code)
if tree is None:
@@ -8420,6 +8467,15 @@ def _resolve_durable_stages(
return _ResolvedDurableStages(result.codes, False, result.empty_reason)
+def _log_durable_stage_split_without_extraction_boundary(*, block_label: str, stage_count: int) -> None:
+ """The one fused-draft shape the sibling diagnostic cannot see: stages derived, no boundary to split at."""
+ LOG.info(
+ "copilot_durable_stage_split_without_extraction_boundary",
+ block_label=block_label,
+ stage_count=stage_count,
+ )
+
+
def _log_durable_stage_derivation_empty(*, block_label: str, resolved: _ResolvedDurableStages) -> None:
LOG.info(
"copilot_durable_stage_derivation_empty",
@@ -8883,10 +8939,27 @@ def _maybe_impose_synthesized_code_block(
rejection_provenance = list(surface_validation.provenance)
ambiguous_reject_present = any(record.kind == "ambiguous" for record in surface_validation.provenance)
submitted_extraction_suffix = _submitted_suffix_after_synthesized_code(submitted_code, synthesized_spine_code)
- extraction_suffix = submitted_extraction_suffix or synthesized.extraction_code
- extraction_candidate_source = "submitted" if submitted_extraction_suffix else "generated"
+ # A frozen candidate is an extraction the plan derived from a packet that bound every requested
+ # path, so it reads the element the run proved carries the value. The submitted suffix is the
+ # author's guess at the same read; where the two disagree the proven one is taken rather than the
+ # attempt rejected, and the test run is what says whether it holds.
+ plan_extraction_wins = (
+ imposed_candidate is not None
+ and bool(synthesized.extraction_code)
+ and bool(submitted_extraction_suffix)
+ and submitted_extraction_suffix.strip() != synthesized.extraction_code.strip()
+ )
+ extraction_suffix = (
+ synthesized.extraction_code
+ if plan_extraction_wins
+ else (submitted_extraction_suffix or synthesized.extraction_code)
+ )
+ extraction_candidate_source = (
+ "submitted" if submitted_extraction_suffix and not plan_extraction_wins else "generated"
+ )
if (
- submitted_extraction_suffix
+ not plan_extraction_wins
+ and submitted_extraction_suffix
and synthesized.extraction_code
and submitted_extraction_suffix.strip() != synthesized.extraction_code.strip()
):
@@ -9006,11 +9079,41 @@ def _maybe_impose_synthesized_code_block(
and not extraction_suffix
and not _is_submitted_code_synthesized_only(submitted_code, synthesized_spine_code)
)
- append_selected_extraction = (
+ # Read-only code that does not restate the spine is appended to it, whatever the metadata said:
+ # the imposed spine would otherwise replace the extraction the author just wrote with nothing.
+ append_selected_extraction = not submitted_contains_full_spine and not selected_mutations and not selected_ambiguous
+ # A draft the surface scan cannot partition is kept and tested rather than replaced by a spine
+ # that carries no extraction: the spine cannot register the value the turn was asked for, so
+ # replacing a returning draft with it trades a testable attempt for a certain miss.
+ preserve_unpartitionable_draft = (
metadata_declares_goal_values
- and not submitted_contains_full_spine
- and not selected_mutations
- and not selected_ambiguous
+ and not preserve_submitted_extraction
+ and not append_selected_extraction
+ and not extraction_suffix
+ and bool(selected_ambiguous)
+ and _submission_returns_a_value(submitted_code)
+ and not _is_submitted_code_synthesized_only(submitted_code, synthesized_spine_code)
+ )
+ if preserve_unpartitionable_draft:
+ preserve_submitted_extraction = True
+ repair_context = ctx.last_code_authoring_repair_context
+ repairing_runtime_failure = repair_context is not None and repair_context.reason_code == "runtime_block_failure"
+ scouted_since_offer = len(ctx.scout_trajectory) > (ctx.synthesized_block_offered_trajectory_len or 0)
+ LOG.info(
+ "copilot_imposition_submitted_code_disposition",
+ preserve_submitted_extraction=preserve_submitted_extraction,
+ preserve_unpartitionable_draft=preserve_unpartitionable_draft,
+ selected_ambiguous=sorted(selected_ambiguous),
+ append_selected_extraction=append_selected_extraction,
+ repairing_runtime_failure=repairing_runtime_failure,
+ scouted_since_offer=scouted_since_offer,
+ metadata_declares_goal_values=metadata_declares_goal_values,
+ submitted_contains_full_spine=submitted_contains_full_spine,
+ has_extraction_suffix=bool(extraction_suffix),
+ submitted_is_synthesized_only=_is_submitted_code_synthesized_only(submitted_code, synthesized_spine_code),
+ # Whether an author's extraction was dropped is unanswerable once the submission is replaced,
+ # so record its shape rather than the code, which carries credential references.
+ **_submitted_code_fingerprint(submitted_code),
)
scrubbed_selected_metadata_label = (
str(code_block.get("label") or "")
@@ -9058,12 +9161,18 @@ def _maybe_impose_synthesized_code_block(
split_eligible = metadata_declares_goal_values or resolved_stages.segmented
if split_eligible and len(durable_stage_codes) < 2:
_log_durable_stage_derivation_empty(block_label=str(code_block.get("label") or ""), resolved=resolved_stages)
+ has_extraction_boundary = bool(extraction_suffix) or append_selected_extraction or preserve_submitted_extraction
+ if split_eligible and len(durable_stage_codes) >= 2 and not has_extraction_boundary:
+ _log_durable_stage_split_without_extraction_boundary(
+ block_label=str(code_block.get("label") or ""),
+ stage_count=len(durable_stage_codes),
+ )
# A credential fill in the trajectory opens the split on its own: a value-producing subset that
# carries no credential fill reruns on the scout's authenticated session instead of replaying login.
should_split_output_owner = (
split_eligible
and len(durable_stage_codes) >= 2
- and (bool(extraction_suffix) or append_selected_extraction or preserve_submitted_extraction)
+ and has_extraction_boundary
and not _output_owner_is_referenced(parsed, str(code_block.get("label") or "").strip())
)
if should_split_output_owner:
@@ -9077,13 +9186,28 @@ def _maybe_impose_synthesized_code_block(
)
split_extraction_code = _apply_parameter_reconciliation_to_code(split_extraction_code, parameter_reconciliation)
target_metadata = _metadata_item_for_block_label(raw_metadata, str(code_block.get("label") or ""))
- required_split_paths = _metadata_item_goal_value_paths(target_metadata)
+ required_split_paths = _goal_paths_rooted_as_the_code_returns_them(
+ _metadata_item_goal_value_paths(target_metadata), split_extraction_code
+ )
if required_split_paths:
split_extraction_code, static_return_violations = _extraction_code_with_value_bearing_static_return(
split_extraction_code,
required_paths=required_split_paths,
)
if static_return_violations:
+ # Which of the extraction candidates this branch was handed, by shape rather than by
+ # code: the submission carries credential references, and the question a refusal here
+ # raises is only ever whether the derived plan's read was available and chosen.
+ LOG.info(
+ "copilot_split_extraction_static_return_refused",
+ plan_present=extraction_plan is not None,
+ plan_produces_paths=sorted(_code_block_produced_output_paths(synthesized.extraction_code)),
+ suffix_is_plan_extraction=bool(synthesized.extraction_code)
+ and extraction_suffix.strip() == synthesized.extraction_code.strip(),
+ has_submitted_suffix=bool(submitted_extraction_suffix),
+ required_split_paths=sorted(required_split_paths),
+ produced_paths=sorted(_code_block_produced_output_paths(split_extraction_code)),
+ )
return _SynthesizedCodeImpositionResult(
workflow_yaml=workflow_yaml,
violations=static_return_violations,
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index b655b8da7..59a5501a3 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -7353,3 +7353,132 @@ def test_an_element_state_timeout_does_not_send_the_user_to_check_the_url() -> N
assert _FAILURE_FOLLOW_UP.get("ELEMENT_STATE_TIMEOUT", "") == ""
assert "confirm the URL" in _FAILURE_FOLLOW_UP["PAGE_LOAD_TIMEOUT"]
+
+
+def test_offer_reopens_once_when_the_requested_read_becomes_provable_after_authoring() -> None:
+ # The page carrying the requested value is often reached after the draft exists (SKY-13226: the
+ # dashboard sits behind a login). Without reopening, the proven read is derived and never offered,
+ # and the draft's own guess is what the first test run executes.
+ criterion = CompletionCriterion(id="record_id", outcome="Record Identifier", output_path="output.record_id")
+ ctx = _ctx()
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ ctx.request_policy = RequestPolicy(completion_criteria=[criterion])
+ ctx.completion_criteria_turn_state = SimpleNamespace(decision=SimpleNamespace(criteria=(criterion,)))
+ ctx.copilot_config = CopilotConfig(requested_output_path_aliases={"record identifier": "output.record_id"})
+ ctx.scout_trajectory = [
+ {"tool_name": "click", "selector": "#show-details", "source_url": "https://example.com/provider"}
+ ]
+ ctx.flow_evidence = []
+ ctx.update_workflow_called = True
+
+ before_the_page_was_reached = _synthesized_block_offer_prompt(ctx)
+ ctx.flow_evidence = [
+ {
+ "step": 2,
+ "reached_via": "interaction",
+ "had_bounded_schema": True,
+ "evidence": {
+ "source_tool": "scout_interaction",
+ "interaction_tool": "click",
+ "interaction_selector": "#show-details",
+ "inspection_warnings": [],
+ "result_containers_truncated": False,
+ "key_value_relations_truncated": False,
+ "key_value_relations": [
+ {
+ "key_text": "Record Identifier",
+ "container_selector": ".record-kv",
+ "container_match_count": 1,
+ "container_position": 0,
+ "value_child_index": 1,
+ "direct_child_count": 2,
+ "visible": True,
+ "value_visible": True,
+ }
+ ],
+ "result_containers": [],
+ },
+ }
+ ]
+ once_provable = _synthesized_block_offer_prompt(ctx)
+ again = _synthesized_block_offer_prompt(ctx)
+
+ assert before_the_page_was_reached == ""
+ assert 'page.locator(".record-kv").nth(0)' in once_provable
+ assert again == ""
+
+
+def test_offer_reopens_for_candidates_when_the_pages_wording_binds_no_plan() -> None:
+ # Live shape (SKY-13226): a log query labelled its count "logs found" while the request said
+ # "azure", so no plan binds. The relations the page does offer are named only by the offer this
+ # gate guards, so requiring a bound plan to reopen asked the read that makes it bind to exist first.
+ criterion = CompletionCriterion(id="c0", outcome="azure", output_path="output.errors")
+ ctx = _ctx()
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ ctx.request_policy = RequestPolicy(completion_criteria=[criterion])
+ ctx.completion_criteria_turn_state = SimpleNamespace(decision=SimpleNamespace(criteria=(criterion,)))
+ ctx.scout_trajectory = [{"tool_name": "click", "selector": "#go", "source_url": "https://example.test/logs"}]
+ ctx.update_workflow_called = True
+ ctx.flow_evidence = [
+ {
+ "step": 1,
+ "reached_via": "current_page",
+ "had_bounded_schema": True,
+ "evidence": {
+ "source_tool": "inspect_page_for_composition",
+ "inspection_warnings": [],
+ "result_containers_truncated": False,
+ "key_value_relations_truncated": False,
+ "key_value_relations": [
+ {
+ "key_text": "logs found",
+ "value_text": "1.22K",
+ "container_selector": ".agg",
+ "container_match_count": 1,
+ "container_position": 0,
+ "value_child_index": 0,
+ "direct_child_count": 2,
+ "visible": True,
+ "value_visible": True,
+ }
+ ],
+ "result_containers": [],
+ },
+ }
+ ]
+
+ offer = _synthesized_block_offer_prompt(ctx)
+
+ assert "logs found = 1.22K" in offer
+ assert "output.errors" in offer
+
+
+def test_a_turn_that_forms_no_requested_output_still_dumps_its_trajectory(tmp_path, monkeypatch) -> None:
+ # A login or navigation defect lives in a turn that never forms a requested output, and dumping
+ # only where a plan was attempted left exactly those turns unreproducible offline (SKY-13226).
+ from skyvern.forge.sdk.copilot import enforcement as enforcement_module
+
+ monkeypatch.setenv("COPILOT_DUMP_DERIVATION_INPUTS", str(tmp_path))
+ monkeypatch.setattr(enforcement_module.settings, "ENV", "local")
+ ctx = _ctx()
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ ctx.request_policy = RequestPolicy(completion_criteria=[])
+ ctx.scout_trajectory = [
+ {
+ "tool_name": "fill_credential_field",
+ "selector": "#totp",
+ "credential_id": "cred_1",
+ "credential_field": "totp",
+ "source_url": "https://example.test/login/totp",
+ }
+ ]
+ ctx.flow_evidence = []
+
+ _synthesized_block_offer_prompt(ctx)
+
+ dumps = list(tmp_path.glob("derivation-no_requested_output-*.json"))
+ assert dumps, "a turn with no requested output wrote no dump"
+ import json
+
+ recorded = json.loads(dumps[0].read_text())["scout_trajectory"]
+ assert [step["credential_field"] for step in recorded] == ["totp"]
diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.py
index fd76a5148..34cd5faf3 100644
--- a/tests/unit/test_copilot_code_block_persist_seam.py
+++ b/tests/unit/test_copilot_code_block_persist_seam.py
@@ -3110,7 +3110,7 @@ class TestCodeRepairProgressClassification:
result = await _update_workflow({"workflow_yaml": workflow_yaml}, ctx, allow_missing_credentials=True)
- assert result["ok"] is True
+ assert result["ok"] is True, result
parsed = parse_workflow_yaml(ctx.workflow_yaml)
assert isinstance(parsed, dict)
code = str(_single_code_block(parsed)["code"])
@@ -5974,6 +5974,89 @@ class TestCompiledAuthoringImposition:
assert 'records = [{"number": "REC-001", "status": "credentialed"}]' in output_code
assert 'return {"records": records}' in output_code
+ @pytest.mark.asyncio
+ async def test_a_returning_draft_the_surface_scan_cannot_partition_is_kept_over_the_spine(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ # A live draft named its locators, which the surface scan reports as ambiguous, so neither
+ # preservation path applied and the spine replaced an extraction the run needed to register.
+ _stub_successful_update(monkeypatch)
+ ctx = self._provider_search_ctx()
+ submitted_code = (
+ 'field = page.locator("#provInput")\n'
+ "await field.fill(str(provider_name))\n"
+ 'value = await page.inner_text("#rec")\n'
+ 'return {"records": [{"number": value}]}\n'
+ )
+ _mutations, _unscouted, ambiguous = workflow_update_module._browser_surface_for_code(submitted_code)
+ assert ambiguous, "fixture must reproduce the unpartitionable shape"
+
+ submitted = yaml.safe_dump(
+ {
+ "title": "Provider lookup",
+ "workflow_definition": {
+ "parameters": [
+ {
+ "parameter_type": "workflow",
+ "workflow_parameter_type": "string",
+ "key": "provider_name",
+ "default_value": "Sample Search",
+ }
+ ],
+ "blocks": [{"block_type": "code", "label": "search_registry", "code": submitted_code}],
+ },
+ },
+ sort_keys=False,
+ )
+ metadata = [_terminal_metadata("search_registry", "search the registry")]
+
+ result = await _update_workflow({"workflow_yaml": submitted, "code_artifact_metadata": metadata}, ctx)
+
+ assert result["ok"] is True
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ assert isinstance(parsed, dict)
+ persisted = "\n".join(
+ str(block.get("code") or "") for block in workflow_blocks(parsed) if block.get("block_type") == "code"
+ )
+ assert 'return {"records": [{"number": value}]}' in persisted
+ assert 'await page.inner_text("#rec")' in persisted
+
+ @pytest.mark.asyncio
+ async def test_a_read_only_extraction_survives_metadata_that_declares_no_goal_values(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = self._provider_search_ctx()
+ synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
+ assert synthesized is not None
+ extraction = 'records = [{"number": await page.inner_text("#rec")}]\nreturn {"records": records}\n'
+ submitted = yaml.safe_dump(
+ {
+ "title": "Provider lookup",
+ "workflow_definition": {
+ "parameters": [
+ {
+ "parameter_type": "workflow",
+ "workflow_parameter_type": "string",
+ "key": "provider_name",
+ "default_value": "Sample Search",
+ }
+ ],
+ "blocks": [{"block_type": "code", "label": "search_registry", "code": extraction}],
+ },
+ },
+ sort_keys=False,
+ )
+
+ result = await _update_workflow({"workflow_yaml": submitted}, ctx)
+
+ assert result["ok"] is True
+ persisted = "\n".join(
+ str(block.get("code") or "") for block in _code_blocks(parse_workflow_yaml(ctx.workflow_yaml)).values()
+ )
+ assert "#provInput" in persisted, "the demonstrated spine must survive"
+ assert "#rec" in persisted, "the model's extraction must survive"
+
def _login_otp_ctx(self) -> CopilotContext:
login_url = "https://example.com/login"
ctx = _code_only_ctx()
@@ -11739,6 +11822,38 @@ def _commit_only_submitted_yaml() -> str:
)
+class TestBoundExtractionPlanIsAnsweredWith:
+ def test_a_plan_that_bound_survives_evidence_that_no_longer_derives_one(self) -> None:
+ # A live turn derived once and degraded on the next sixteen evaluations, so the bound plan was
+ # gone by the imposition that needed it and the generated read invented its own locator.
+ ctx = _live_read_extraction_ctx()
+
+ bound = enforcement_module.requested_output_extraction_plan(ctx)
+
+ assert bound is not None
+ assert ctx.last_bound_requested_output_extraction_plan is bound
+
+ ctx.flow_evidence = []
+
+ assert enforcement_module.requested_output_extraction_plan(ctx) is bound
+
+ def test_a_retained_plan_is_abandoned_once_the_request_asks_for_other_paths(self) -> None:
+ ctx = _live_read_extraction_ctx()
+ assert enforcement_module.requested_output_extraction_plan(ctx) is not None
+
+ ctx.flow_evidence = []
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(id="other", outcome="Other Field", output_path="output.other_field")
+ ]
+ )
+ ctx.completion_criteria_turn_state = SimpleNamespace(
+ decision=SimpleNamespace(criteria=tuple(ctx.request_policy.completion_criteria))
+ )
+
+ assert enforcement_module.requested_output_extraction_plan(ctx) is None
+
+
class TestCommitOnlyReach:
def test_commit_only_two_click_trajectory_reaches_without_output_binding(self) -> None:
ctx = _commit_only_ctx()
@@ -12346,6 +12461,23 @@ def test_producer_table_with_sibling_declaration_emits_per_row_none_leaf() -> No
assert violations == []
+def test_sole_returned_local_binds_to_the_one_required_path_whatever_it_is_named() -> None:
+ emitted, violations = workflow_update_module._extraction_code_with_required_static_return(
+ 'visitors = await page.inner_text(".metric")\nreturn visitors',
+ required_paths={"output.visitor_count"},
+ )
+ assert violations == []
+ assert 'return {"output": {"visitor_count": visitors}}' in emitted
+
+
+def test_a_renamed_local_still_needs_a_single_unambiguous_candidate() -> None:
+ _, violations = workflow_update_module._extraction_code_with_required_static_return(
+ 'visitors = await page.inner_text(".a")\ndates = await page.inner_text(".b")\nreturn visitors',
+ required_paths={"output.visitor_count", "output.date_range"},
+ )
+ assert violations
+
+
def test_producer_abstains_on_array_declaration_without_matching_table_group() -> None:
plan = _scalar_plan(_scalar_binding("output.account_number", "Account Number", ".acct"))
envelope = produce_covered_static_return_envelope(
@@ -12415,6 +12547,9 @@ def _download_classification_ctx(
scouted_output_covered_paths=set(covered_paths or set()),
reached_download_target=download,
flow_evidence=[],
+ scout_trajectory=[],
+ requested_output_designations=[],
+ composition_page_evidence=None,
)
@@ -13841,6 +13976,53 @@ class TestScoutCredentialSegments:
},
]
+ def test_the_value_read_runs_outside_the_login_presence_guard(self) -> None:
+ # SKY-13226: the guard exists so an authenticated replay skips the login, but it indented
+ # every later step, so the read never ran and the block returned a name it never bound
+ # (UnboundLocalError: _read_value_0, live runs M1/N1/Y1).
+ url = "https://example.com/web"
+ trajectory = [
+ {
+ "tool_name": "fill_credential_field",
+ "selector": "#user",
+ "source_url": url,
+ "credential_id": "cred_1",
+ "credential_field": "username",
+ },
+ {
+ "tool_name": "fill_credential_field",
+ "selector": "#pass",
+ "source_url": url,
+ "credential_id": "cred_1",
+ "credential_field": "password",
+ },
+ {
+ "tool_name": "click",
+ "selector": "#sign-in",
+ "role": "button",
+ "accessible_name": "Sign in",
+ "source_url": url,
+ },
+ {
+ "tool_name": "read_value",
+ "read_expression": "document.querySelector('#total').textContent",
+ "read_output_path": "output.total",
+ "source_url": url,
+ },
+ ]
+
+ synthesized = workflow_update_module.synthesize_code_block(trajectory, strict_selectors=True)
+
+ assert synthesized is not None
+ lines = synthesized.code.splitlines()
+ guard = next(index for index, line in enumerate(lines) if ".count() == 1:" in line)
+ read = next(index for index, line in enumerate(lines) if "_read_value_0 = await" in line)
+ guard_indent = len(lines[guard]) - len(lines[guard].lstrip())
+
+ assert read > guard
+ assert (len(lines[read]) - len(lines[read].lstrip())) <= guard_indent
+ ast.parse(textwrap.dedent(synthesized.code).strip())
+
def test_segments_are_valid_self_contained_python(self) -> None:
synthesized = workflow_update_module.synthesize_code_block(self._credential_trajectory(), strict_selectors=True)
assert synthesized is not None
@@ -14054,3 +14236,67 @@ class TestScoutCredentialSegments:
assert "inner_text" in owner_code
for code in (*stage_codes, owner_code):
ast.parse(textwrap.dedent(code).strip())
+
+ @pytest.mark.asyncio
+ async def test_fused_draft_without_extraction_boundary_names_the_missing_boundary(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+
+ async def _no_credential_validation_error(_value: object, _ctx: object) -> None:
+ return None
+
+ monkeypatch.setattr(
+ workflow_update_module, "_credential_reference_validation_error", _no_credential_validation_error
+ )
+ ctx = _code_only_ctx()
+ _enable_imposition(ctx)
+ ctx.scout_trajectory = self._credential_trajectory()[:4]
+ ctx.scouted_credential_field_inventory_by_credential_id = {"cred_1": frozenset({"username", "password"})}
+ synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
+ assert synthesized is not None
+ # Same draft as the split case minus the extraction suffix: stages derive, nothing to split at.
+ submitted = yaml.safe_dump(
+ {
+ "title": "Dashboard total",
+ "workflow_definition": {
+ "parameters": [],
+ "blocks": [{"block_type": "code", "label": "get_total", "code": synthesized.code}],
+ },
+ },
+ sort_keys=False,
+ )
+ metadata = [_no_goal_value_metadata("get_total", "read the dashboard total")]
+
+ with capture_logs() as logs:
+ result = await _update_workflow({"workflow_yaml": submitted, "code_artifact_metadata": metadata}, ctx)
+
+ assert result["ok"] is True
+ emitted = [
+ entry for entry in logs if entry.get("event") == "copilot_durable_stage_split_without_extraction_boundary"
+ ]
+ assert len(emitted) == 1
+ assert emitted[0]["stage_count"] >= 2
+ assert emitted[0]["block_label"] == "get_total"
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ assert isinstance(parsed, dict)
+ labels = [str(block.get("label") or "") for block in workflow_blocks(parsed)]
+ assert not any("browser_stage" in label for label in labels)
+
+
+def test_a_goal_path_is_rooted_where_the_plan_returns_the_requested_output_namespace() -> None:
+ # Live shape (SKY-13226): the plan's extraction returned {"output": {"visitors": ...}} while the
+ # block metadata named the same value "visitors", so imposition refused code that returned it.
+ rooted = workflow_update_module._goal_paths_rooted_as_the_code_returns_them(
+ {"visitors"}, 'return {"output": {"visitors": _extraction_value_0}}'
+ )
+
+ assert rooted == {"output.visitors"}
+
+
+def test_a_goal_path_stays_bare_when_the_block_returns_a_bare_mapping() -> None:
+ bare = workflow_update_module._goal_paths_rooted_as_the_code_returns_them(
+ {"records"}, 'return {"records": records}'
+ )
+
+ assert bare == {"records"}
diff --git a/tests/unit/test_copilot_code_block_synthesis.py b/tests/unit/test_copilot_code_block_synthesis.py
index 7b5c397eb..facdfe46d 100644
--- a/tests/unit/test_copilot_code_block_synthesis.py
+++ b/tests/unit/test_copilot_code_block_synthesis.py
@@ -5274,6 +5274,63 @@ class TestScoutedReadSynthesis:
assert block.code.rstrip().endswith('return {"error_count": _read_value_0}')
ast.parse(textwrap.dedent(block.code))
+ def test_emitted_read_fails_loudly_when_the_value_never_appears(self) -> None:
+ # The scout only records a read that returned something. An empty replay contradicts that
+ # proof, so the block must end rather than register a field carrying nothing.
+ block = synthesize_code_block(
+ [
+ {
+ "tool_name": "read_value",
+ "read_expression": 'document.querySelector("#total").innerText',
+ "read_output_path": "output.error_count",
+ "read_result_shape": "str",
+ "source_url": "https://example.test/logs",
+ "trajectory_index": 0,
+ },
+ ]
+ )
+ assert block is not None
+ code = textwrap.dedent(block.code)
+ ast.parse(code)
+
+ settle = next(line.strip() for line in code.splitlines() if line.strip().startswith("if _read_value_0 not in"))
+ decide = eval(f"lambda _read_value_0: bool({settle.removeprefix('if ').rstrip(':')})") # noqa: S307
+ for empty in (None, "", [], {}):
+ assert not decide(empty), f"emitted read settled for {empty!r}"
+ for value in ("8.3K", 0, 0.0, ["row"], {"a": 1}):
+ assert decide(value), f"emitted read discarded {value!r}"
+
+ # and the block raises rather than returning the empty value
+ raises = [line.strip() for line in code.splitlines() if line.strip().startswith("raise Exception")]
+ assert raises, "an unresolved read must end the block"
+ assert "output.error_count" in raises[0]
+
+ def test_an_empty_replay_contradicts_the_proof_whatever_shape_it_had(self) -> None:
+ # A read is only recorded once it returns something, so there are no empty proven reads: an
+ # empty replay is absence for a collection-proven read exactly as for a scalar-proven one.
+ for shape in ("str", "list", "dict"):
+ block = synthesize_code_block(
+ [
+ {
+ "tool_name": "read_value",
+ "read_expression": 'document.querySelectorAll("tr")',
+ "read_output_path": "output.rows",
+ "read_result_shape": shape,
+ "source_url": "https://example.test/rows",
+ "trajectory_index": 0,
+ }
+ ]
+ )
+ assert block is not None
+ code = textwrap.dedent(block.code)
+ ast.parse(code)
+ guard = next(line.strip() for line in code.splitlines() if line.strip().startswith("if _read_value_0 in"))
+ decide = eval(f"lambda _read_value_0: bool({guard.removeprefix('if ').rstrip(':')})") # noqa: S307
+ for empty in (None, "", [], {}):
+ assert decide(empty), f"{shape}-proven read must treat {empty!r} as absent"
+ for value in ("8.3K", 0, ["row"], {"a": 1}):
+ assert not decide(value), f"{shape}-proven read must accept {value!r}"
+
def test_read_without_an_expression_is_dropped_rather_than_emitted_empty(self) -> None:
block = synthesize_code_block(
[
@@ -5447,3 +5504,108 @@ def test_synthesized_extraction_guards_only_name_things_the_sandbox_provides() -
diagnostic = sandbox_unresolved_name_repair_diagnostic("\n".join(emitted), parameter_keys=[])
assert diagnostic is None, diagnostic.unresolved_names if diagnostic else ()
+
+
+def test_a_value_witnessed_read_proves_no_heading_at_runtime() -> None:
+ # Live failure (SKY-13226): the value witness picks the element by the value it showed, then the
+ # generated read asserted the heading beside it and raised "Observed scalar label changed" —
+ # the tile's heading slot also carries a delta chip, and a page with no heading has none to read.
+ from skyvern.forge.sdk.copilot.code_block_synthesis import _key_value_scalar_read_statements
+ from skyvern.forge.sdk.copilot.output_extraction_plan import LiveReadBinding
+
+ fields = {
+ "output_path": "output.count",
+ "kind": "key_value_scalar",
+ "selector": "#summary",
+ "selector_count": 1,
+ "selector_index": 0,
+ "child_index": 1,
+ "child_count": 3,
+ "relation_label": "Visitors",
+ }
+
+ witnessed = "\n".join(
+ _key_value_scalar_read_statements(
+ LiveReadBinding(**fields, identified_by_label=False), "_value", guard_empty=True
+ )
+ )
+ by_label = "\n".join(_key_value_scalar_read_statements(LiveReadBinding(**fields), "_value", guard_empty=True))
+
+ assert "Observed scalar label" not in witnessed
+ # The element is still pinned by everything that chose it.
+ assert "Observed scalar selector cardinality changed" in witnessed
+ assert "Observed scalar direct-child shape changed" in witnessed
+ assert '_value = (await page.locator("#summary").nth(0).locator(":scope > *").nth(1).inner_text()).strip()' in (
+ witnessed
+ )
+ # A binding the label did choose keeps proving it.
+ assert "Observed scalar label changed" in by_label
+
+
+def test_entry_anchors_on_what_the_flow_fills_not_the_link_it_clicks_away_from() -> None:
+ # Live capture (SKY-13226): a scout opened a password-reset link before signing in, and the login
+ # block anchored on that link — so every execution began by leaving the login form.
+ trajectory = [
+ {
+ "tool_name": "click",
+ "selector": 'a[href="/account/forgot_password"]',
+ "accessible_name": "Forgot password?",
+ "source_url": "https://example.test/account/login",
+ },
+ {
+ "tool_name": "fill_credential_field",
+ "selector": "#username",
+ "credential_id": "cred_1",
+ "credential_field": "username",
+ "source_url": "https://example.test/account/login",
+ },
+ {
+ "tool_name": "fill_credential_field",
+ "selector": "#password",
+ "credential_id": "cred_1",
+ "credential_field": "password",
+ "source_url": "https://example.test/account/login",
+ },
+ ]
+
+ block = synthesize_code_block(trajectory, strict_selectors=True)
+
+ assert block is not None
+ assert '_scout_entry_target = page.locator("#username")' in block.code
+ # The click is still demonstrated, it just no longer decides where the flow starts.
+ assert 'a[href=\\"/account/forgot_password\\"]").click()' in block.code
+ assert [step["action_type"] for step in block.steps][:4] == ["goto_url", "click", "input_text", "input_text"]
+
+
+def test_a_designated_read_waits_for_a_figure_the_page_has_not_rendered_yet() -> None:
+ # Live failure (SKY-13226): the tile renders while its query is still resolving, so a read that
+ # fires once takes the empty frame — and on the log explorer it took the neighbouring control's
+ # text instead, costing a whole test run.
+ from skyvern.forge.sdk.copilot.code_block_synthesis import _key_value_scalar_read_statements
+ from skyvern.forge.sdk.copilot.output_extraction_plan import LiveReadBinding
+
+ designated = LiveReadBinding(
+ output_path="output.count",
+ kind="key_value_scalar",
+ selector="#total",
+ selector_count=1,
+ selector_index=0,
+ )
+ emitted = "\n".join(_key_value_scalar_read_statements(designated, "_value", guard_empty=True))
+
+ assert "wait_for_timeout" in emitted
+ assert emitted.count("inner_text()") == 2, "the value is re-read after each wait, not just once"
+ assert 'raise Exception("Designated value element is empty")' in emitted
+
+ # A relation-bound read proves its own shape and is left alone.
+ relation = LiveReadBinding(
+ output_path="output.count",
+ kind="key_value_scalar",
+ selector="#card",
+ selector_count=1,
+ selector_index=0,
+ child_index=1,
+ child_count=3,
+ relation_label="Visitors",
+ )
+ assert "wait_for_timeout" not in "\n".join(_key_value_scalar_read_statements(relation, "_value", guard_empty=True))
diff --git a/tests/unit/test_copilot_completion_criteria_lifecycle.py b/tests/unit/test_copilot_completion_criteria_lifecycle.py
index 33225c873..5344acce1 100644
--- a/tests/unit/test_copilot_completion_criteria_lifecycle.py
+++ b/tests/unit/test_copilot_completion_criteria_lifecycle.py
@@ -327,6 +327,17 @@ def test_reconcile_no_criteria_anywhere_is_noop() -> None:
),
id="antecedent-family",
),
+ pytest.param(
+ (
+ CompletionCriterion(
+ id="__copilot_requested_output__output_visitors_last_7_days",
+ outcome="the number of visitors in the last 7 days is output",
+ output_path="output.visitors_last_7_days",
+ requested_output_label="visitors",
+ ),
+ ),
+ id="requested-output-label",
+ ),
],
)
def test_criteria_json_round_trip_preserves_fields(criteria: tuple[CompletionCriterion, ...]) -> None:
diff --git a/tests/unit/test_copilot_composition_evidence.py b/tests/unit/test_copilot_composition_evidence.py
index 202b336de..8da29c865 100644
--- a/tests/unit/test_copilot_composition_evidence.py
+++ b/tests/unit/test_copilot_composition_evidence.py
@@ -28,7 +28,10 @@ from skyvern.forge.sdk.copilot.composition_browser_expressions import (
COMPOSITION_VISUAL_OBSTRUCTION_CANDIDATES_EXPRESSION,
)
from skyvern.forge.sdk.copilot.composition_evidence import (
+ _BARE_MAGNITUDE_RE,
_MAX_CLICKABLE_CONTROLS,
+ _MAX_SELECTOR_CHARS,
+ _structural_path,
composition_page_evidence_error,
has_actionable_steer_content,
has_bounded_page_schema,
@@ -39,6 +42,7 @@ from skyvern.forge.sdk.copilot.composition_evidence import (
parse_composition_html,
parse_composition_structured,
)
+from skyvern.forge.sdk.copilot.output_extraction_plan import _relation_label_child_index
from skyvern.forge.sdk.copilot.result_evidence import (
loaded_result_composition_evidence_from_page,
loaded_result_composition_target_summary,
@@ -2528,6 +2532,37 @@ def test_html_reveal_shape_multi_value_leaves_carry_empty_key() -> None:
assert all(r["key_text"] == "" for r in reveal)
+def test_html_reveal_shape_metric_tile_keys_the_magnitude_leaf_by_heading() -> None:
+ parsed = parse_composition_html(
+ '<body><section id="result"><h3>Visitors</h3>'
+ "<div>-15.0%</div><div>8.45K</div><div>vs. 9.99K prior</div></section></body>",
+ inspected_url=_REVEAL_URL,
+ current_url=_REVEAL_URL,
+ )
+ # The metric-card pass supersedes the reveal shape for this tile: one keyed relation pairing
+ # the heading with the magnitude leaf, no unkeyed delta remainders.
+ reveal = [r for r in parsed["key_value_relations"] if r["direct_child_count"] == 4]
+ assert [(r["key_text"], r["value_text"], r["value_child_index"]) for r in reveal] == [("Visitors", "8.45K", 2)]
+
+
+def test_html_reveal_shape_two_magnitude_leaves_stay_unkeyed() -> None:
+ parsed = parse_composition_html(
+ '<body><section id="result"><h3>Visitors</h3>'
+ "<div>8.45K</div><div>9.99K</div><div>vs. prior</div></section></body>",
+ inspected_url=_REVEAL_URL,
+ current_url=_REVEAL_URL,
+ )
+ reveal = [r for r in parsed["key_value_relations"] if r["direct_child_count"] == 4]
+ assert len(reveal) == 3
+ assert all(r["key_text"] == "" for r in reveal)
+
+
+def test_browser_reveal_shape_designates_the_same_leaf_as_the_html_parser() -> None:
+ assert "valueLeaves.length === 1 ? keyText : ''" not in COMPOSITION_STRUCTURED_EVIDENCE_EXPRESSION
+ assert "leaf.index === designatedIndex ? keyText : ''" in COMPOSITION_STRUCTURED_EVIDENCE_EXPRESSION
+ assert _BARE_MAGNITUDE_RE.pattern in COMPOSITION_STRUCTURED_EVIDENCE_EXPRESSION
+
+
def test_html_reveal_shape_rejects_structural_and_token_negatives() -> None:
def reveal_relations(inner: str, container: str = 'id="result"') -> list[dict[str, Any]]:
parsed = parse_composition_html(
@@ -2575,7 +2610,7 @@ def test_html_reveal_shape_page_cap_emits_truncation_signal() -> None:
inner = "<h3>Heading</h3>" + "".join(f"<div>v{i}</div>" for i in range(5))
containers = "".join(f'<section id="result-{index}">{inner}</section>' for index in range(5))
parsed = parse_composition_html(
- f"<body><header><h1>Title</h1><p>Sub</p></header>{containers}</body>",
+ f"<body><div class=intro><h1>Title</h1><p>Sub</p></div>{containers}</body>",
inspected_url=_REVEAL_URL,
current_url=_REVEAL_URL,
)
@@ -2592,7 +2627,7 @@ def test_html_reveal_shape_non_truncating_multi_reveal_emits_no_warning() -> Non
inner = "<h3>Heading</h3><div>A</div><p>B</p>"
containers = "".join(f'<section id="result-{index}">{inner}</section>' for index in range(3))
parsed = parse_composition_html(
- f"<body><header><h1>Title</h1><p>Sub</p></header>{containers}</body>",
+ f"<body><div class=intro><h1>Title</h1><p>Sub</p></div>{containers}</body>",
inspected_url=_REVEAL_URL,
current_url=_REVEAL_URL,
)
@@ -3695,3 +3730,495 @@ def test_has_witnessed_value_content_false_on_blank_value_text() -> None:
packet = _kv_value_content_packet()
packet["key_value_relations"][0]["value_text"] = " "
assert has_witnessed_value_content(packet) is False
+
+
+_METRIC_DASHBOARD_HTML = """
+<body>
+ <nav id="sidebar">
+ <div><span>Visitors</span><span>3</span></div>
+ <div><span>I</span><span>x</span></div>
+ </nav>
+ <main>
+ <div id="visitors-card">
+ <div><span>Visitors</span><span>-8.5%</span></div>
+ <div>8.83K</div>
+ <div>vs 9.58K prior</div>
+ </div>
+ <div id="sessions-card">
+ <div><span>Sessions</span><span>-10.0%</span></div>
+ <div>10.7K</div>
+ <div>vs 11.8K prior</div>
+ </div>
+ </main>
+</body>
+"""
+
+
+def test_metric_card_pairs_the_heading_with_the_magnitude_not_the_delta() -> None:
+ # Live custody (SKY-13226/SKY-13332): the only capturable "Visitors" pair was the header row,
+ # which pairs the label with the -8.5% delta; the 8.83K figure was structurally invisible.
+ parsed = parse_composition_html(
+ _METRIC_DASHBOARD_HTML, inspected_url="https://example.test/web", current_url="https://example.test/web"
+ )
+ visitors = [r for r in parsed["key_value_relations"] if r["key_text"] == "Visitors"]
+ assert [(r["value_text"], r["value_child_index"]) for r in visitors] == [("8.83K", 1)]
+ assert parsed["key_value_relations_truncated"] is False
+
+
+def test_page_chrome_relations_are_excluded() -> None:
+ # The sidebar "Visitors" nav item would shadow the tile (ambiguous binding) and chrome pairs
+ # flood the relation cap into a truncation flag that voids the whole packet.
+ parsed = parse_composition_html(
+ _METRIC_DASHBOARD_HTML, inspected_url="https://example.test/web", current_url="https://example.test/web"
+ )
+ keys = [r["key_text"] for r in parsed["key_value_relations"]]
+ assert keys.count("Visitors") == 1
+ assert "I" not in keys
+
+
+def test_metric_dashboard_derives_a_grounded_plan_end_to_end() -> None:
+ # AC1's offline witness: the captured packet binds the requested label to the tile value.
+ from skyvern.forge.sdk.copilot.output_extraction_plan import derive_requested_output_extraction_plan
+
+ parsed = parse_composition_html(
+ _METRIC_DASHBOARD_HTML, inspected_url="https://example.test/web", current_url="https://example.test/web"
+ )
+ entry = {"step": 4, "reached_via": "current_page", "had_bounded_schema": True, "evidence": parsed}
+ plan = derive_requested_output_extraction_plan(
+ flow_evidence=[entry], labels_by_path={"output.visitors": ("visitors",)}
+ )
+ assert plan is not None
+ (binding,) = plan.live_reads
+ assert binding.relation_label == "Visitors"
+ assert binding.child_index == 1
+
+
+def test_browser_twin_carries_the_metric_card_and_chrome_guards() -> None:
+ assert "metricCardNodes" in COMPOSITION_STRUCTURED_EVIDENCE_EXPRESSION
+ assert "insidePageChrome" in COMPOSITION_STRUCTURED_EVIDENCE_EXPRESSION
+ assert "nonContentChildTags" in COMPOSITION_STRUCTURED_EVIDENCE_EXPRESSION
+ assert "nav,aside,header,footer,[role=navigation]" in COMPOSITION_STRUCTURED_EVIDENCE_EXPRESSION
+ assert COMPOSITION_STRUCTURED_EVIDENCE_EXPRESSION.count("const bareMagnitude") == 1
+
+
+def test_no_packet_selector_is_a_truncated_fragment() -> None:
+ # A selector cut to a length budget can end mid-token, and the block generated from it raises
+ # SyntaxError from querySelectorAll on a selector that never matched anything. A control with no
+ # id or name falls through to the class chain, which is where selectors outgrow the bound.
+ classes = " ".join(f"ui-analytics-dashboard-control-surface-element-variant-{i}" for i in range(6))
+ nesting = "".join(f'<div class="wrapper-layer-with-a-fairly-long-name-{i}">' for i in range(8))
+ parsed = parse_composition_html(
+ f'<html><body><form action="/search">{nesting}'
+ f'<input type="text" class="{classes}" /><button type="submit" class="{classes}">Go</button>'
+ f"{'</div>' * 8}</form></body></html>",
+ inspected_url="https://analytics.example.test/web",
+ current_url="https://analytics.example.test/web",
+ )
+
+ def selectors(node: Any) -> list[str]:
+ found: list[str] = []
+ if isinstance(node, dict):
+ for key, value in node.items():
+ if key == "selector" and isinstance(value, str):
+ found.append(value)
+ else:
+ found.extend(selectors(value))
+ elif isinstance(node, list):
+ for item in node:
+ found.extend(selectors(item))
+ return found
+
+ emitted = selectors(parsed)
+ assert emitted, "expected the packet to describe at least one control"
+ for selector in emitted:
+ # An absent selector is fine — a fragment is not.
+ assert len(selector) != 160, f"selector sits exactly at the bound, so it was cut: {selector!r}"
+ if not selector:
+ continue
+ assert selector.count("[") == selector.count("]"), f"unbalanced attribute selector: {selector!r}"
+ assert selector.count("(") == selector.count(")"), f"unbalanced pseudo-class: {selector!r}"
+ assert not selector.endswith((":", "-", "_", ".", "#", ",", ">")), f"truncated selector: {selector!r}"
+
+
+def test_no_normalized_browser_selector_is_a_truncated_fragment() -> None:
+ # The same invariant on the packet the live browser returns, which is the one that reaches a
+ # generated block. A tile nested deeply in id-less markup overruns the bound, and cutting it to
+ # fit produced "div:nth-of-ty" — a selector that parses nowhere and reads as a product failure.
+ long_selector = " > ".join(["div:nth-of-type(3)"] * 9)
+ payload: dict[str, Any] = {
+ "page_title": "Dashboard",
+ "forms": [],
+ "clickable_controls": [{"text": "Go", "selector": long_selector, "tag": "button"}],
+ "navigation_targets": [],
+ "result_containers": [],
+ "modal_overlays": [],
+ "page_obstructions": [],
+ "challenge_controls": [],
+ "key_value_relations": [
+ {
+ "key_text": "Visitors",
+ "value_text": "8.7K",
+ "container_selector": long_selector,
+ "container_match_count": 1,
+ "container_position": 0,
+ "value_child_index": 1,
+ "direct_child_count": 2,
+ "visible": True,
+ "value_visible": True,
+ }
+ ],
+ }
+
+ parsed = parse_composition_structured(payload, inspected_url="https://example.test/web", current_url="u")
+
+ assert parsed is not None
+ emitted = [relation.get("container_selector", "") for relation in parsed.get("key_value_relations") or []] + [
+ control.get("selector", "") for control in parsed.get("clickable_controls") or []
+ ]
+ for selector in emitted:
+ assert len(selector) != 160, f"selector sits exactly at the bound, so it was cut: {selector!r}"
+ assert not selector.endswith((":", "-", "_", ".", "#", ",", ">")), f"truncated selector: {selector!r}"
+
+
+def test_a_deep_path_that_overruns_the_bound_falls_back_to_a_shorter_unique_tail() -> None:
+ # Rejecting the overrun keeps a broken selector out of a block, but on its own it also drops the
+ # only relation carrying the requested value. The shortest tail that still resolves to this node
+ # alone is the read that survives the bound.
+ from bs4 import BeautifulSoup
+
+ tile = '<section><article><div id="tile"><span>Visitors</span><span>8.7K</span></div></article></section>'
+ body = tile
+ for _ in range(6):
+ body = f"<div><div>a</div><div>{body}</div></div>"
+ soup = BeautifulSoup(f"<body>{body}</body>", "html.parser")
+ node = soup.find(id="tile")
+ del node["id"]
+
+ path = _structural_path(node)
+
+ assert len(path) <= _MAX_SELECTOR_CHARS
+ assert path.endswith("article:nth-of-type(1) > div:nth-of-type(1)")
+ assert len(soup.select(path)) == 1
+
+
+def test_a_metric_tile_pairs_its_heading_with_the_figure_not_the_delta() -> None:
+ # A live dashboard capture recorded 'Visitors' -> '-17.0%' and filed the figure under the
+ # comparison text, so the requested number was absent from the evidence the binder was handed.
+ from bs4 import BeautifulSoup
+
+ from skyvern.forge.sdk.copilot.composition_evidence import _key_value_relations
+
+ soup = BeautifulSoup(
+ """<div id="root"><div class="tile">
+ <div class="hdr"><span>Visitors</span><span>-17.0%</span></div>
+ <div class="fig"><span>8.7K</span><span>vs. 9.48K prior</span></div>
+ </div></div>""",
+ "html.parser",
+ )
+
+ relations, _truncated, _reveal_truncated = _key_value_relations(soup)
+
+ assert [(relation["key_text"], relation["value_text"]) for relation in relations] == [("Visitors", "8.7K")]
+ relation = relations[0]
+ carrier = soup.select(relation["container_selector"])[relation["container_position"]]
+ children = [child for child in carrier.find_all(recursive=False) if child.name]
+ assert children[relation["value_child_index"]].get_text(" ", strip=True) == "8.7K"
+
+
+_DEEP_TILE_HTML = (
+ "<body><main><div id=tile>"
+ "<div><span>Visitors</span><span>-17.0%</span></div>"
+ "<div><div><span>7.89K</span></div></div>"
+ "<div>vs. 9.55K prior</div>"
+ "</div></main></body>"
+)
+
+
+def test_a_requested_label_reaches_the_figure_its_tile_nests_deeper() -> None:
+ # Live capture on a real dashboard recorded ('Visitors', '-17.0%'): the shape passes give up once
+ # a grandchild has children of its own, and the tile's header row - heading beside delta badge -
+ # was captured instead and read as the requested value.
+ parsed = parse_composition_html(
+ _DEEP_TILE_HTML,
+ inspected_url="https://example.test/web",
+ current_url="https://example.test/web",
+ requested_targets=("Visitors",),
+ )
+
+ visitors = [r for r in parsed["key_value_relations"] if r["key_text"] == "Visitors"]
+
+ assert [r["value_text"] for r in visitors] == ["7.89K"]
+
+
+def test_a_tile_reaches_its_figure_whether_or_not_the_label_was_requested() -> None:
+ # The targeted pass rescued this shape only for labels the turn named, so a tile whose label was
+ # minted as prose kept the delta badge and offered it as the page's answer. A decorated delta is
+ # decoration either way, so the figure is reached without depending on the mint (SKY-13226).
+ parsed = parse_composition_html(
+ _DEEP_TILE_HTML, inspected_url="https://example.test/web", current_url="https://example.test/web"
+ )
+
+ visitors = [r for r in parsed["key_value_relations"] if r["key_text"] == "Visitors"]
+
+ assert [r["value_text"] for r in visitors] == ["7.89K"]
+
+
+def test_a_requested_label_abstains_when_its_tile_carries_several_candidate_figures() -> None:
+ # Markup that cannot say which number the label owns is not evidence that any of them is.
+ parsed = parse_composition_html(
+ "<body><main><div id=tile>"
+ "<div><span>Visitors</span></div>"
+ "<div><div><span>7.89K</span></div></div>"
+ "<div><div><span>9.55K</span></div></div>"
+ "</div></main></body>",
+ inspected_url="https://example.test/web",
+ current_url="https://example.test/web",
+ requested_targets=("Visitors",),
+ )
+
+ assert [r["value_text"] for r in parsed["key_value_relations"] if r["key_text"] == "Visitors"] != ["7.89K"]
+
+
+def test_browser_twin_resolves_requested_targets_before_the_shape_passes() -> None:
+ from skyvern.forge.sdk.copilot.composition_browser_expressions import (
+ composition_structured_evidence_expression,
+ )
+
+ expression = composition_structured_evidence_expression(("Visitors",))
+
+ assert '"Visitors"' in expression.split("const ANTI_BOT_PATTERNS")[0]
+ assert "valueBesideLabel" in expression
+ # The targeted pass must own its carrier before the metric-card loop can claim it.
+ assert expression.index("for (const target of REQUESTED_TARGETS)") < expression.index("const magnitudeLeaves2")
+ assert "REQUESTED_TARGETS=[]" in composition_structured_evidence_expression()
+
+
+def test_a_generated_read_proves_a_non_sibling_label_at_its_own_anchor() -> None:
+ from skyvern.forge.sdk.copilot.code_block_synthesis import synthesize_extraction_suffix
+ from skyvern.forge.sdk.copilot.output_extraction_plan import derive_requested_output_extraction_plan
+
+ parsed = parse_composition_html(
+ _DEEP_TILE_HTML,
+ inspected_url="https://example.test/web",
+ current_url="https://example.test/web",
+ requested_targets=("Visitors",),
+ )
+ plan = derive_requested_output_extraction_plan(
+ flow_evidence=[{"step": 1, "reached_via": "current_page", "had_bounded_schema": True, "evidence": parsed}],
+ labels_by_path={"output.visitors": ("Visitors",)},
+ )
+
+ assert plan is not None
+ binding = plan.live_reads[0]
+ assert binding.label_selector
+ suffix = synthesize_extraction_suffix(plan)
+ assert suffix is not None
+ # The label is proven where it lives, not as the value's first sibling.
+ assert f'page.locator("{binding.label_selector}").first.inner_text()' in suffix.code
+
+
+def test_browser_twin_carries_the_non_sibling_label_anchor() -> None:
+ from skyvern.forge.sdk.copilot.composition_browser_expressions import (
+ composition_structured_evidence_expression,
+ )
+
+ expression = composition_structured_evidence_expression(("Visitors",))
+
+ assert "label_selector: labelSelector" in expression
+ assert "resolvesUniquely(labelSelector, labelEl)" in expression
+
+
+_VALUE_FIRST_TILE_HTML = "<body><main><div id=agg><span>1.22K</span><span>logs found</span></div></main></body>"
+
+
+def test_a_tile_that_prints_its_figure_before_its_label_records_where_the_label_is() -> None:
+ # Live capture (SKY-13226): a log query renders "1.22K logs found", so the value is the first
+ # child and a read proving the label at child zero raises on a page that plainly shows both.
+ parsed = parse_composition_html(
+ _VALUE_FIRST_TILE_HTML, inspected_url="https://example.test/logs", current_url="https://example.test/logs"
+ )
+
+ relation = next(r for r in parsed["key_value_relations"] if r["key_text"] == "logs found")
+
+ assert (relation["value_text"], relation["value_child_index"], relation["label_child_index"]) == ("1.22K", 0, 1)
+
+
+def test_a_value_first_tile_binds_by_witness_and_reads_it_without_proving_a_heading() -> None:
+ from skyvern.forge.sdk.copilot.code_block_synthesis import synthesize_extraction_suffix
+ from skyvern.forge.sdk.copilot.output_extraction_plan import derive_requested_output_extraction_plan
+
+ parsed = parse_composition_html(
+ _VALUE_FIRST_TILE_HTML, inspected_url="https://example.test/logs", current_url="https://example.test/logs"
+ )
+ plan = derive_requested_output_extraction_plan(
+ flow_evidence=[{"step": 1, "reached_via": "current_page", "had_bounded_schema": True, "evidence": parsed}],
+ labels_by_path={"output.errors": ("azure",)},
+ witnessed_by_path={"output.errors": "1.22K"},
+ )
+
+ # "azure" shares no wording with "logs found": the witnessed quantity is what joins them.
+ assert plan is not None
+ binding = plan.live_reads[0]
+ assert (binding.child_index, binding.label_child_index) == (0, 1)
+ code = synthesize_extraction_suffix(plan).code
+ # The witness chose this element by the value it showed, so re-reading the heading proves nothing
+ # about that choice — and a tile whose heading slot also carries a delta chip, or a page with no
+ # heading at all, has nothing to re-read (SKY-13226).
+ assert "logs found" not in code
+ assert "nth(0).inner_text()).strip()\n" in code or "nth(0).inner_text()).strip()" in code
+
+
+def _playwright_chromium_available() -> bool:
+ try:
+ from playwright.sync_api import sync_playwright # noqa: PLC0415
+
+ with sync_playwright() as runner:
+ browser = runner.chromium.launch()
+ browser.close()
+ return True
+ except Exception:
+ return False
+
+
+_ICON_SPRITE_SHEET_PAGE = (
+ "<html><body><svg style='display:none' aria-hidden='true'>"
+ + "".join(
+ f'<symbol id="icons_arrow-{name}_{index:04x}--sprite"><path d="M0 0h1v1H0z"/></symbol>'
+ for index, name in enumerate(
+ ("center-horizontal", "center-vertical", "down", "end-bottom", "end-left", "end-right", "up", "start")
+ )
+ )
+ + "</svg>"
+ + '<div class="query-results-summary"><span>1.31K</span><span>logs found</span></div>'
+ + "</body></html>"
+)
+
+
+ not _playwright_chromium_available(),
+ reason="Requires Playwright browsers installed (run: playwright install chromium)",
+)
+async def test_browser_twin_spends_the_container_budget_on_containers_a_value_can_live_in() -> None:
+ # The twins have to agree about what a result container is, and only this one runs against a real
+ # page: an icon sprite sheet sits at the top of the document, so a substring test that reads "row"
+ # out of "arrow" claims every slot of the bounded budget before any real result is reached.
+ from playwright.async_api import async_playwright # noqa: PLC0415
+
+ async with async_playwright() as runner:
+ browser = await runner.chromium.launch()
+ try:
+ page = await browser.new_page()
+ await page.set_content(_ICON_SPRITE_SHEET_PAGE)
+ raw = await page.evaluate(COMPOSITION_STRUCTURED_EVIDENCE_EXPRESSION)
+ finally:
+ await browser.close()
+
+ live = json.loads(raw) if isinstance(raw, str) else raw
+ captured = live["result_containers"]
+ assert [entry["tag"] for entry in captured if entry["tag"] == "symbol"] == []
+ assert not live["result_containers_truncated"]
+ assert [entry["tag"] for entry in captured] == ["div"]
+
+ parsed = parse_composition_html(
+ _ICON_SPRITE_SHEET_PAGE,
+ inspected_url="https://example.test/logs",
+ current_url="https://example.test/logs",
+ )
+ assert [entry["tag"] for entry in parsed["result_containers"]] == [entry["tag"] for entry in captured]
+
+
+def test_a_dialog_only_observation_does_not_shadow_the_tile_captured_before_it() -> None:
+ from skyvern.forge.sdk.copilot.output_extraction_plan import derive_requested_output_extraction_plan
+ from skyvern.forge.sdk.copilot.tools._shared import _append_flow_evidence
+
+ def parsed(body: str) -> dict[str, Any]:
+ return parse_composition_html(
+ f'<html><body><nav><a href="/logs">Logs</a></nav>{body}</body></html>',
+ inspected_url="https://example.test/logs",
+ current_url="https://example.test/logs",
+ )
+
+ tile = parsed('<div class="query-results-summary"><span>1.22K</span><span>logs found</span></div>')
+ dialog = parsed(
+ '<div role="dialog" aria-label="Update your time zone">'
+ '<div class="dialog-footer"><button>No, keep it</button><button>Yes, update</button></div></div>'
+ )
+ ctx = SimpleNamespace(flow_evidence=[])
+ _append_flow_evidence(ctx, tile, reached_via="current_page")
+ _append_flow_evidence(ctx, dialog, reached_via="current_page")
+
+ # Derivation spends its one attempt on the freshest bindable packet, so a capture describing
+ # nothing but the dialog in front takes that attempt and the tile behind it is never read.
+ plan = derive_requested_output_extraction_plan(
+ flow_evidence=ctx.flow_evidence,
+ labels_by_path={"output.errors": ("azure",)},
+ witnessed_by_path={"output.errors": "1.22K"},
+ )
+
+ assert plan is not None
+ assert [binding.relation_label for binding in plan.live_reads] == ["logs found"]
+ assert [entry["obstructed"] for entry in ctx.flow_evidence] == [False, True]
+
+
+def test_a_decorated_delta_does_not_cost_the_tile_its_figure() -> None:
+ # Live shape (SKY-13226): the delta renders as an arrow plus its text, so treating any nested
+ # grandchild as structure abandoned the whole tile and the positional builder paired the heading
+ # with the delta — the figure sat at the next child the entire time.
+ html = """
+ <div><div>
+ <div><span>Visitors</span><div><svg></svg><span>-12.0%</span></div></div>
+ <div>8.43K</div>
+ <div>vs. 9.55K prior</div>
+ </div></div>
+ """
+
+ packet = parse_composition_html(html, inspected_url="https://example.test/a", current_url="https://example.test/a")
+
+ visitors = [r for r in packet["key_value_relations"] if r["key_text"] == "Visitors"]
+ assert [(r["value_text"], r["value_child_index"], r["direct_child_count"]) for r in visitors] == [("8.43K", 1, 3)]
+
+
+def test_a_genuinely_nested_subtree_is_still_not_a_tile() -> None:
+ html = """
+ <div><div>
+ <div><span>Visitors</span><div><span>a</span><span>b</span></div></div>
+ <div>8.43K</div>
+ </div></div>
+ """
+
+ packet = parse_composition_html(html, inspected_url="https://example.test/a", current_url="https://example.test/a")
+
+ assert not [r for r in packet["key_value_relations"] if r["value_text"] == "8.43K"]
+
+
+def test_the_page_side_capture_keeps_a_label_that_sits_outside_the_value_row() -> None:
+ payload = {
+ "page_title": "Logs",
+ "forms": [],
+ "navigation_targets": [],
+ "result_containers": [],
+ "result_containers_truncated": False,
+ "key_value_relations": [
+ {
+ "key_text": "logs found",
+ "value_text": "1.42K",
+ "container_selector": ".tile > .row",
+ "container_match_count": 1,
+ "container_position": 0,
+ "value_child_index": 0,
+ "label_child_index": -1,
+ "direct_child_count": 2,
+ "visible": True,
+ "value_visible": True,
+ }
+ ],
+ }
+
+ packet = parse_composition_structured(
+ payload, inspected_url="https://example.test/logs", current_url="https://example.test/logs"
+ )
+
+ relation = packet["key_value_relations"][0]
+ assert relation["label_child_index"] == -1
+ assert _relation_label_child_index(relation) == -1
diff --git a/tests/unit/test_copilot_enforcement_hardening.py b/tests/unit/test_copilot_enforcement_hardening.py
index f5d96b5c2..8c058ebd9 100644
--- a/tests/unit/test_copilot_enforcement_hardening.py
+++ b/tests/unit/test_copilot_enforcement_hardening.py
@@ -17,6 +17,7 @@
from __future__ import annotations
import json
+from types import SimpleNamespace
from typing import Any
import pytest
@@ -52,6 +53,7 @@ from skyvern.forge.sdk.copilot.enforcement import (
_recover_from_context_overflow,
_scouted_spine_missing_text,
_strip_input_images,
+ _witnessed_values_by_path,
register_no_progress_interaction_click,
reset_no_progress_interaction_count,
synthesized_trajectory_reaches_goal,
@@ -589,3 +591,50 @@ def test_repair_obligation_releases_the_turn_once_its_rounds_are_spent() -> None
ctx.repair_obligation_nudge_count = MAX_REPAIR_OBLIGATION_NUDGES
assert _check_enforcement(ctx, result) is None
+
+
+class TestWitnessArbitration:
+ def _ctx(self, reads: list[str]) -> SimpleNamespace:
+ packet = {
+ "source_tool": "inspect_page_for_composition",
+ "key_value_relations": [
+ {"key_text": "logs found", "value_text": "1.41K", "visible": True, "value_visible": True}
+ ],
+ }
+ return SimpleNamespace(
+ flow_evidence=[{"evidence": packet, "reached_via": "current_page", "had_bounded_schema": True, "step": 0}],
+ scout_trajectory=[
+ {
+ "tool_name": "read_value",
+ "read_output_path": "output.azure_error_count",
+ "read_output_path_source": "declared",
+ "read_result_value": value,
+ }
+ for value in reads
+ ],
+ )
+
+ def test_conflicting_reads_resolve_to_the_value_the_page_still_shows(self) -> None:
+ # Live shape (SKY-13226): a login-form probe and the real read both claimed the path; dropping
+ # the pair left the one requested output with no witness at all.
+ ctx = self._ctx(['{"passwordId":"password"}', "1.41K"])
+ assert _witnessed_values_by_path(ctx) == {"output.azure_error_count": "1.41K"}
+
+ def test_a_conflict_the_page_corroborates_for_none_still_carries_no_witness(self) -> None:
+ ctx = self._ctx(["junk-a", "junk-b"])
+ assert _witnessed_values_by_path(ctx) == {}
+
+ def test_a_read_that_only_inherited_the_path_witnesses_nothing(self) -> None:
+ # An early probe of a login form was promoted to the requested output because it was the only
+ # path on offer, and its JSON then stood as the witness for a figure the page had not shown.
+ ctx = self._ctx([])
+ ctx.scout_trajectory = [
+ {
+ "tool_name": "read_value",
+ "read_output_path": "output.azure_error_count",
+ "read_output_path_source": "elimination",
+ "read_result_value": '{"passwordId": "password"}',
+ }
+ ]
+
+ assert _witnessed_values_by_path(ctx) == {}
diff --git a/tests/unit/test_copilot_enforcement_pruning.py b/tests/unit/test_copilot_enforcement_pruning.py
index 86b17e7e1..83e1201ac 100644
--- a/tests/unit/test_copilot_enforcement_pruning.py
+++ b/tests/unit/test_copilot_enforcement_pruning.py
@@ -66,6 +66,7 @@ from skyvern.forge.sdk.copilot.enforcement import (
run_with_enforcement,
synthesized_block_persistence_signal,
synthesized_goal_completion_landing_pending,
+ synthesized_offer_reopened_for_extraction_plan,
synthesized_persistence_reopened,
synthesized_persistence_reopened_after_failed_run,
synthesized_trajectory_is_goal_complete,
@@ -148,6 +149,8 @@ class _Ctx:
self.scout_observed_terminal_criterion_ids: set[str] = set()
self.scout_observation_contract: object | None = None
self.flow_evidence: list[dict[str, object]] = []
+ self.last_bound_requested_output_extraction_plan = None
+ self.requested_output_designations: list[dict[str, object]] = []
self.composition_page_evidence = None
self.copilot_config: CopilotConfig | None = None
self.latest_recorded_build_test_outcome = None
@@ -4417,3 +4420,67 @@ def test_total_timeout_override_binds_on_settings_and_defaults_unset() -> None:
def test_shared_tools_bind_the_configured_total_timeout() -> None:
assert shared_total_timeout_seconds == TOTAL_TIMEOUT_SECONDS
assert TOTAL_TIMEOUT_SECONDS == (settings.WORKFLOW_COPILOT_TOTAL_TIMEOUT_SECONDS or 900)
+
+
+class TestUnboundOfferReopen:
+ """The offer names relations no read has claimed, so it stops once one of them is read."""
+
+ @staticmethod
+ def _ctx_on_a_page_with_relations() -> _Ctx:
+ ctx = _Ctx()
+ ctx.scout_trajectory = []
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(id="c0", outcome="the number of visitors", output_path="output.visitors")
+ ]
+ )
+ ctx.flow_evidence = [
+ {
+ "step": 1,
+ "reached_via": "current_page",
+ "had_bounded_schema": True,
+ "evidence": {
+ "source_tool": "inspect_page_for_composition",
+ "inspection_warnings": [],
+ "result_containers": [],
+ "result_containers_truncated": False,
+ "key_value_relations_truncated": False,
+ "key_value_relations": [
+ {
+ "key_text": "Visitors",
+ "value_text": "7.82K",
+ "container_selector": ".card",
+ "container_match_count": 1,
+ "container_position": 0,
+ "value_child_index": 1,
+ "direct_child_count": 3,
+ "visible": True,
+ "value_visible": True,
+ }
+ ],
+ },
+ }
+ ]
+ return ctx
+
+ def test_an_unread_requested_output_reopens_the_offer(self) -> None:
+ ctx = self._ctx_on_a_page_with_relations()
+
+ assert synthesized_offer_reopened_for_extraction_plan(ctx, None) is True
+
+ def test_a_read_that_answered_the_path_stops_reopening_it(self) -> None:
+ ctx = self._ctx_on_a_page_with_relations()
+ ctx.scout_trajectory = [
+ {
+ "tool_name": "read_value",
+ "read_expression": "document.querySelector('.card .val').innerText",
+ "read_output_path": "output.visitors",
+ "read_output_path_source": "declared",
+ "read_result_value": "7.82K",
+ "read_result_shape": "str",
+ }
+ ]
+
+ # Without the latch this stays True for the rest of the turn and the whole offer is
+ # re-emitted on every prompt build.
+ assert synthesized_offer_reopened_for_extraction_plan(ctx, None) is False
diff --git a/tests/unit/test_copilot_evaluate_page_evidence.py b/tests/unit/test_copilot_evaluate_page_evidence.py
index d8f3ab971..7774360ab 100644
--- a/tests/unit/test_copilot_evaluate_page_evidence.py
+++ b/tests/unit/test_copilot_evaluate_page_evidence.py
@@ -365,6 +365,225 @@ async def test_scouted_read_binds_to_a_canonical_slot_when_that_is_the_requested
assert [read["read_output_path"] for read in reads] == [slot_path]
+async def test_each_read_binds_to_the_requested_output_it_names() -> None:
+ # Counting requested outputs can only attribute a read when the turn requests exactly one, so a
+ # request for several fields bound none of them and returned them under an anonymous path.
+ from skyvern.forge.sdk.copilot.tools.mcp_hooks import _evaluate_pre_hook
+
+ ctx = _ctx()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(id="c0", outcome="the visitor count", output_path="output.visitors"),
+ CompletionCriterion(id="c1", outcome="the error count", output_path="output.errors"),
+ ]
+ )
+
+ for expression, output_path in (
+ ("document.querySelector('#visitors').textContent", "output.visitors"),
+ ("document.querySelector('#errors').textContent", "output.errors"),
+ ):
+ await _evaluate_pre_hook({"expression": expression, "output_path": output_path}, ctx)
+ await _evaluate_post_hook(
+ {"ok": True, "data": {"result": "8.45K", "url": "https://dash.example.test/web"}},
+ raw={"name": "evaluate"},
+ ctx=ctx,
+ )
+
+ reads = [i for i in ctx.scout_trajectory if i.get("tool_name") == "read_value"]
+ assert [read["read_output_path"] for read in reads] == ["output.visitors", "output.errors"]
+ assert ctx.pending_scout_read_output_path is None
+
+
+async def test_an_empty_read_is_not_recorded_as_a_proven_read() -> None:
+ from skyvern.forge.sdk.copilot.tools.mcp_hooks import _evaluate_pre_hook
+
+ for empty in ("", [], {}, None):
+ ctx = _ctx()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[CompletionCriterion(id="c0", outcome="the count", output_path="output.count")]
+ )
+ await _evaluate_pre_hook({"expression": "document.querySelector('#c').textContent"}, ctx)
+ await _evaluate_post_hook(
+ {"ok": True, "data": {"result": empty, "url": "https://dash.example.test/"}},
+ raw={"name": "evaluate"},
+ ctx=ctx,
+ )
+ assert [i for i in ctx.scout_trajectory if i.get("tool_name") == "read_value"] == [], empty
+
+
+async def test_a_markup_dump_is_not_recorded_as_the_read_for_a_scalar_output() -> None:
+ # The live defect: the scout inspects a card by returning its outerHTML, and that probe becomes
+ # the read synthesis replays as the extraction for output.visitors.
+ from skyvern.forge.sdk.copilot.tools.mcp_hooks import _evaluate_pre_hook
+
+ ctx = _ctx()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(id="c0", outcome="the number of visitors", output_path="output.visitors")
+ ]
+ )
+ markup = '<div class="card"><span>Visitors</span><span>8.3K</span></div>' * 40
+
+ await _evaluate_pre_hook({"expression": "document.querySelector('.card').outerHTML.slice(0, 12000)"}, ctx)
+ await _evaluate_post_hook(
+ {"ok": True, "data": {"result": markup, "url": "https://dash.example.test/web"}},
+ raw={"name": "evaluate"},
+ ctx=ctx,
+ )
+
+ reads = [i for i in ctx.scout_trajectory if i.get("tool_name") == "read_value"]
+ bound = [r for r in reads if r.get("read_output_path") == "output.visitors"]
+ assert bound == [], "a markup dump must not become the proven read for a scalar requested output"
+
+
+async def test_a_scalar_read_keeps_the_value_it_saw_and_a_dump_does_not() -> None:
+ # Only the type name was kept, so nothing downstream could locate the element still carrying the
+ # value the scout had already read.
+ from skyvern.forge.sdk.copilot.tools.mcp_hooks import _evaluate_pre_hook
+
+ async def _read(ctx: CopilotContext, expression: str, result: object) -> None:
+ await _evaluate_pre_hook({"expression": expression, "output_path": "output.visitors"}, ctx)
+ await _evaluate_post_hook(
+ {"ok": True, "data": {"result": result, "url": "https://dash.example.test/web"}},
+ raw={"name": "evaluate"},
+ ctx=ctx,
+ )
+
+ ctx = _ctx()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(id="c0", outcome="the number of visitors", output_path="output.visitors")
+ ]
+ )
+
+ await _read(ctx, "document.querySelector('.tile .value').innerText", "8.7K")
+ await _read(ctx, "document.querySelector('.card')", {"tag": "div", "children": 4})
+ await _read(ctx, "document.body.innerText", "x" * 400)
+
+ reads = [item for item in ctx.scout_trajectory if item.get("tool_name") == "read_value"]
+ assert [item.get("read_result_value") for item in reads] == ["8.7K", None, None]
+
+
+async def test_sole_requested_output_still_claims_a_read_that_named_another_purpose() -> None:
+ # Two runs registered visitors_last_week=8700 through exactly this path: the reader named its own
+ # purpose and elimination still attributed the read to the one output the turn was asked for.
+ # Diverting that read to the scouted-read slot left every later turn registering nothing.
+ from skyvern.forge.sdk.copilot.tools.mcp_hooks import _evaluate_pre_hook
+
+ ctx = _ctx()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(id="c0", outcome="the number of visitors", output_path="output.visitors")
+ ]
+ )
+
+ await _evaluate_pre_hook(
+ {"expression": "document.querySelector('.card .value').innerText", "output_path": "visitor_card_value"}, ctx
+ )
+ await _evaluate_post_hook(
+ {"ok": True, "data": {"result": "8.7K", "url": "https://dash.example.test/web"}},
+ raw={"name": "evaluate"},
+ ctx=ctx,
+ )
+
+ reads = [item for item in ctx.scout_trajectory if item.get("tool_name") == "read_value"]
+ assert [item.get("read_output_path") for item in reads] == ["output.visitors"]
+
+
+async def test_evaluate_names_the_requested_output_no_read_has_claimed() -> None:
+ # A live turn probed the tile's structure three times, each read naming its own purpose, and
+ # reached authoring with nothing bound to the output it was asked for.
+ from skyvern.forge.sdk.copilot.tools.mcp_hooks import _evaluate_pre_hook
+
+ # Two requested outputs, so elimination cannot attribute a read on its own and a probe that
+ # names its own purpose leaves both slots unclaimed.
+ ctx = _ctx()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(id="c0", outcome="the number of visitors", output_path="output.visitors"),
+ CompletionCriterion(id="c1", outcome="the number of sessions", output_path="output.sessions"),
+ ]
+ )
+
+ await _evaluate_pre_hook(
+ {"expression": "document.querySelector('.card')", "output_path": "visitor_card_structure"}, ctx
+ )
+ probe = await _evaluate_post_hook(
+ {"ok": True, "data": {"result": {"tag": "div"}, "url": "https://dash.example.test/web"}},
+ raw={"name": "evaluate"},
+ ctx=ctx,
+ )
+ assert probe["data"]["requested_outputs_still_unread"] == ["output.sessions", "output.visitors"]
+
+ await _evaluate_pre_hook(
+ {"expression": "document.querySelector('.card .value').innerText", "output_path": "output.visitors"}, ctx
+ )
+ answered = await _evaluate_post_hook(
+ {"ok": True, "data": {"result": "8.3K", "url": "https://dash.example.test/web"}},
+ raw={"name": "evaluate"},
+ ctx=ctx,
+ )
+ assert answered["data"]["requested_outputs_still_unread"] == ["output.sessions"]
+
+
+async def test_a_declared_read_holding_no_single_value_leaves_its_output_unread() -> None:
+ # Claiming on the declaration alone deleted the outstanding-output signal, so the turn was told
+ # its read recorded nothing and that nothing was outstanding, in the same reply (SKY-13226).
+ from skyvern.forge.sdk.copilot.tools.mcp_hooks import _evaluate_pre_hook
+
+ ctx = _ctx()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(id="c0", outcome="the number of visitors", output_path="output.visitors"),
+ CompletionCriterion(id="c1", outcome="the number of sessions", output_path="output.sessions"),
+ ]
+ )
+
+ await _evaluate_pre_hook(
+ {"expression": "document.querySelectorAll('.card')", "output_path": "output.visitors"}, ctx
+ )
+ gathered = await _evaluate_post_hook(
+ {"ok": True, "data": {"result": [{"t": "8.3K"}, {"t": "12"}], "url": "https://dash.example.test/web"}},
+ raw={"name": "evaluate"},
+ ctx=ctx,
+ )
+
+ assert gathered["data"]["claimed_output_without_a_single_value"] == "output.visitors"
+ assert gathered["data"]["requested_outputs_still_unread"] == ["output.sessions", "output.visitors"]
+
+
+async def test_a_read_naming_an_output_the_request_never_asked_for_is_not_bound_to_it() -> None:
+ from skyvern.forge.sdk.copilot.tools.mcp_hooks import _evaluate_pre_hook
+
+ ctx = _ctx()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(id="c0", outcome="the visitor count", output_path="output.visitors"),
+ CompletionCriterion(id="c1", outcome="the error count", output_path="output.errors"),
+ ]
+ )
+
+ await _evaluate_pre_hook(
+ {"expression": "document.querySelector('#other').textContent", "output_path": "output.invented"}, ctx
+ )
+ await _evaluate_post_hook(
+ {"ok": True, "data": {"result": "12", "url": "https://dash.example.test/web"}},
+ raw={"name": "evaluate"},
+ ctx=ctx,
+ )
+
+ reads = [i for i in ctx.scout_trajectory if i.get("tool_name") == "read_value"]
+ assert [read["read_output_path"] for read in reads] == ["output.scouted_read"]
+
+
@pytest.mark.asyncio
async def test_a_later_diagnostic_read_does_not_evict_the_requested_output_read() -> None:
# Reads sharing an output path collapse to the last one, so a page dump taken after the value
diff --git a/tests/unit/test_copilot_mcp_adapter.py b/tests/unit/test_copilot_mcp_adapter.py
new file mode 100644
index 000000000..3c0f6e94f
--- /dev/null
+++ b/tests/unit/test_copilot_mcp_adapter.py
@@ -0,0 +1,63 @@
+import inspect
+
+from skyvern.forge.sdk.copilot.mcp_adapter import _requested_output_path_choices
+
+
+def _schema() -> dict:
+ return {
+ "type": "object",
+ "properties": {
+ "expression": {"type": "string"},
+ "output_path": {"type": "string", "description": "The requested output this read fills."},
+ },
+ "required": ["expression"],
+ }
+
+
+class TestRequestedOutputPathChoices:
+ def test_the_turn_s_requested_paths_become_the_choices(self) -> None:
+ # Live shape (SKY-13226): a free-form path let the model name its own purpose, so the read
+ # that saw the requested quantity was filed as exploration and never witnessed the path.
+ schema = _requested_output_path_choices(_schema(), ["output.azure_error_count"])
+
+ assert schema["properties"]["output_path"]["enum"] == ["output.azure_error_count"]
+ assert "output.azure_error_count" in schema["properties"]["output_path"]["description"]
+
+ def test_exploration_still_passes_by_omitting_the_path(self) -> None:
+ schema = _requested_output_path_choices(_schema(), ["output.azure_error_count"])
+
+ assert "output_path" not in schema["required"]
+
+ def test_every_requested_path_stays_available_for_a_reread(self) -> None:
+ schema = _requested_output_path_choices(_schema(), ["output.a", "output.b"])
+
+ assert schema["properties"]["output_path"]["enum"] == ["output.a", "output.b"]
+
+ def test_a_turn_owing_no_output_leaves_the_schema_alone(self) -> None:
+ assert _requested_output_path_choices(_schema(), []) == _schema()
+
+
+class TestClaimedOutputWithoutValue:
+ def test_a_declared_read_that_gathered_candidates_is_told_what_to_do_next(self) -> None:
+ # Live shape (SKY-13226): the read named the requested path and returned a filtered list of
+ # every matching line on the page, so no single value was recorded. The bare fact was emitted
+ # six times in one turn and went unacted on; the sibling signal carrying a hint was followed.
+ from skyvern.forge.sdk.copilot.tools import mcp_hooks
+
+ source = inspect.getsource(mcp_hooks)
+ marker = source.index('data["claimed_output_without_a_single_value"]')
+ following = source[marker : marker + 700]
+
+ assert "claimed_output_read_hint" in following
+ assert "output_path=" in following
+
+
+def test_the_declared_path_says_the_expression_is_that_value() -> None:
+ from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
+ from skyvern.forge.sdk.copilot.tools.mcp_hooks import _build_skyvern_mcp_overlays
+
+ overlays = _build_skyvern_mcp_overlays(BlockAuthoringPolicy.CODE_ONLY_BROWSER)
+ description = overlays["evaluate"].copilot_params["output_path"]["description"]
+
+ assert "evaluates to that one value" in description
+ assert "exploration" in description
diff --git a/tests/unit/test_copilot_mcp_adapter_no_progress.py b/tests/unit/test_copilot_mcp_adapter_no_progress.py
index 7a6fc1735..5e376a97b 100644
--- a/tests/unit/test_copilot_mcp_adapter_no_progress.py
+++ b/tests/unit/test_copilot_mcp_adapter_no_progress.py
@@ -69,3 +69,54 @@ async def test_raised_non_click_tool_leaves_no_progress_counter_untouched() -> N
assert result.isError is True
assert ctx.consecutive_no_progress_interaction_count == 0
+
+
+class _HangingClient:
+ async def call_tool(self, name: str, args: dict[str, Any], raise_on_error: bool = False) -> NoReturn:
+ import asyncio
+
+ await asyncio.sleep(3600)
+ raise AssertionError("unreachable")
+
+
+async def test_an_overlay_ceiling_bounds_a_call_that_never_returns() -> None:
+ # Live shape (SKY-13226): an evaluate against a stale session handle answered nothing and held
+ # the turn for 307s of a 900s budget; the overlay's declared ceiling was consumed by no code.
+ ctx = _agent_ctx()
+ server = SkyvernOverlayMCPServer(
+ transport=MagicMock(),
+ overlays={"evaluate": SchemaOverlay(timeout=1)},
+ alias_map={},
+ allowlist=frozenset(),
+ context_provider=lambda: ctx,
+ )
+ server._client = _HangingClient()
+
+ result = await server.call_tool("evaluate", {"expression": "1+1"})
+
+ text = "".join(getattr(block, "text", "") for block in result.content)
+ assert '"ok": false' in text
+ assert "1s" in text
+ # The call is cancelled mid-flight, and click/type_text carry a ceiling too, so the result says
+ # the effect is unknown rather than reporting an action that may have landed as a clean failure.
+ assert "unknown" in text
+
+
+async def test_no_declared_ceiling_means_no_timeout() -> None:
+ import asyncio
+
+ class _SlowClient:
+ async def call_tool(self, name: str, args: dict[str, Any], raise_on_error: bool = False):
+ await asyncio.sleep(0.05)
+ raise RuntimeError("made it past any implicit ceiling")
+
+ ctx = _agent_ctx()
+ server = _make_server(ctx, "get_block_schema")
+ server._client = _SlowClient()
+
+ result = await server.call_tool("get_block_schema", {})
+
+ text = "".join(getattr(block, "text", "") for block in result.content)
+ assert "made it past any implicit ceiling" in text
diff --git a/tests/unit/test_copilot_output_extraction_plan.py b/tests/unit/test_copilot_output_extraction_plan.py
index 2f286bc6b..c22e738b7 100644
--- a/tests/unit/test_copilot_output_extraction_plan.py
+++ b/tests/unit/test_copilot_output_extraction_plan.py
@@ -2,7 +2,7 @@ from __future__ import annotations
from types import SimpleNamespace
-from skyvern.forge.sdk.copilot.composition_evidence import parse_composition_structured
+from skyvern.forge.sdk.copilot.composition_evidence import parse_composition_html, parse_composition_structured
from skyvern.forge.sdk.copilot.mcp_adapter import _restore_post_hook_context, _snapshot_post_hook_context
from skyvern.forge.sdk.copilot.output_extraction_plan import (
LiveReadKind,
@@ -13,10 +13,15 @@ from skyvern.forge.sdk.copilot.output_extraction_plan import (
_key_value_shape_bindings,
_table_shape_bindings,
array_parent_path,
+ bindable_candidate_headings,
+ derivation_bail_reason,
derive_requested_output_extraction_plan,
+ plan_from_designations,
resolve_shape_expectations_by_path,
+ unbound_candidate_relations,
value_matches_shape,
)
+from skyvern.forge.sdk.copilot.tools._shared import _append_flow_evidence
_SYNTHETIC_SHAPE_REGISTRY = {
"widget_id": ShapeExpectation(ValueShape.NUMERIC_ID, ValueCardinality.SCALAR, id_digit_length=8),
@@ -103,13 +108,44 @@ def _flow_packet(*, visible: bool = True, ambiguous_id: bool = False, truncated:
}
-def _derive(*, packet: dict[str, object] | None = None):
+def _derive(*, packet: dict[str, object] | None = None, labels_by_path: dict[str, tuple[str, ...]] | None = None):
return derive_requested_output_extraction_plan(
flow_evidence=[packet or _flow_packet()],
- labels_by_path=LABELS_BY_PATH,
+ labels_by_path=LABELS_BY_PATH if labels_by_path is None else labels_by_path,
)
+def _custody_relation(key_text: str, value_text: str, *, selector: str, child_index: int, child_count: int) -> dict:
+ return {
+ "key_text": key_text,
+ "value_text": value_text,
+ "container_selector": selector,
+ "container_match_count": 1,
+ "container_position": 0,
+ "value_child_index": child_index,
+ "direct_child_count": child_count,
+ "visible": True,
+ "value_visible": True,
+ }
+
+
+def _custody_packet(*relations: dict) -> dict[str, object]:
+ return {
+ "step": 4,
+ "reached_via": "interaction",
+ "had_bounded_schema": True,
+ "evidence": {
+ "source_tool": "scout_interaction",
+ "interaction_selector": "#show-dashboard",
+ "inspection_warnings": [],
+ "result_containers_truncated": False,
+ "key_value_relations_truncated": False,
+ "key_value_relations": list(relations),
+ "result_containers": [],
+ },
+ }
+
+
def test_derives_complete_plan_from_one_visible_interaction_packet() -> None:
plan = _derive()
@@ -126,6 +162,99 @@ def test_derives_complete_plan_from_one_visible_interaction_packet() -> None:
assert plan.identity
+def _navigated_packet(
+ *,
+ source_tool: str = "inspect_page_for_composition",
+ reached_via: str = "current_page",
+ **overrides: object,
+) -> dict[str, object]:
+ packet = _flow_packet()
+ evidence = dict(packet["evidence"], source_tool=source_tool) # type: ignore[call-overload]
+ for key in ("interaction_tool", "interaction_selector"):
+ evidence.pop(key, None)
+ evidence.update(overrides)
+ return {"step": 4, "reached_via": reached_via, "had_bounded_schema": True, "evidence": evidence}
+
+
+def test_deep_link_inspection_stamped_navigate_binds_without_an_anchor() -> None:
+ # A target_url inspection navigates there itself and is stamped "navigate", which is the live
+ # shape of the ticket's deep-link case (wr_557574..., 8 containers / 24 relations, plan underived).
+ plan = derive_requested_output_extraction_plan(
+ flow_evidence=[_navigated_packet(reached_via="navigate")], labels_by_path=LABELS_BY_PATH
+ )
+
+ assert plan is not None
+ assert plan.reveal is None
+
+
+def test_post_run_observation_never_binds_a_plan() -> None:
+ assert (
+ derive_requested_output_extraction_plan(
+ flow_evidence=[_navigated_packet(reached_via="post_run")], labels_by_path=LABELS_BY_PATH
+ )
+ is None
+ )
+
+
+def test_derives_plan_from_a_navigated_page_with_no_reveal_interaction() -> None:
+ plan = derive_requested_output_extraction_plan(flow_evidence=[_navigated_packet()], labels_by_path=LABELS_BY_PATH)
+
+ assert plan is not None
+ assert plan.reveal is None
+ assert {binding.output_path for binding in plan.live_reads} == {
+ "output.record_id",
+ "output.records[].detail",
+ "output.records[].state",
+ "output.overall_state",
+ }
+
+
+def test_navigated_packet_binds_after_login_interactions_precede_it() -> None:
+ login = _flow_packet()
+ login["step"] = 2
+ login["evidence"] = dict(login["evidence"], key_value_relations=[], result_containers=[]) # type: ignore[call-overload]
+ dashboard = _navigated_packet()
+ dashboard["step"] = 9
+
+ plan = derive_requested_output_extraction_plan(flow_evidence=[login, dashboard], labels_by_path=LABELS_BY_PATH)
+
+ assert plan is not None
+ assert plan.observation_step == 9
+ assert plan.reveal is None
+
+
+def test_navigated_packet_keeps_every_completeness_guard() -> None:
+ assert (
+ derive_requested_output_extraction_plan(
+ flow_evidence=[_navigated_packet(result_containers_truncated=True)], labels_by_path=LABELS_BY_PATH
+ )
+ is None
+ )
+ assert (
+ derive_requested_output_extraction_plan(
+ flow_evidence=[_navigated_packet(inspection_warnings=["stale frame"])], labels_by_path=LABELS_BY_PATH
+ )
+ is None
+ )
+ assert (
+ derive_requested_output_extraction_plan(
+ flow_evidence=[_navigated_packet(source_tool="self_heal_verify")], labels_by_path=LABELS_BY_PATH
+ )
+ is None
+ )
+
+
+def test_interaction_packet_without_an_anchor_still_fails_closed() -> None:
+ orphan = _flow_packet()
+ orphan["evidence"] = { # type: ignore[assignment]
+ key: value
+ for key, value in orphan["evidence"].items() # type: ignore[union-attr]
+ if key not in {"interaction_selector", "interaction_role", "interaction_accessible_name"}
+ }
+
+ assert derive_requested_output_extraction_plan(flow_evidence=[orphan], labels_by_path=LABELS_BY_PATH) is None
+
+
def test_identical_reobservation_keeps_structural_candidate_identity() -> None:
first = _flow_packet()
second = _flow_packet()
@@ -204,6 +333,84 @@ def test_hidden_ambiguous_truncated_and_mixed_packets_fail_closed() -> None:
)
+def test_an_entry_carrying_no_schema_is_not_the_one_entry_derivation_spends() -> None:
+ # Live turns bailed entry-unbounded-schema while a bindable packet sat one entry back: the
+ # freshest entry was chosen for having a bindable reached_via, then rejected for having no schema.
+ labels = {"output.visitors": ("Visitors",)}
+ bindable = _bail_entry([_bail_relation("Visitors", selector=".tile")])
+ schemaless = _bail_entry([_bail_relation("Visitors", selector=".tile")])
+ schemaless["had_bounded_schema"] = False
+
+ assert derive_requested_output_extraction_plan(flow_evidence=[bindable], labels_by_path=labels) is not None
+
+ plan = derive_requested_output_extraction_plan(flow_evidence=[bindable, schemaless], labels_by_path=labels)
+
+ assert plan is not None
+ assert [binding.output_path for binding in plan.live_reads] == ["output.visitors"]
+
+
+def test_a_witnessed_value_binds_the_relation_still_showing_it_when_no_label_matches() -> None:
+ # The requested label and the page heading share nothing, which is the shape exact label equality
+ # cannot bind; the value the scout already read identifies the element on its own.
+ labels = {"output.visitors": ("the number of visitors in the last week is returned",)}
+ tile = _bail_relation("Visitors", selector=".tile")
+ tile["value_text"] = "8.7K"
+ delta = _bail_relation("Change", selector=".delta")
+ delta["value_text"] = "-8.0%"
+ entry = _bail_entry([tile, delta])
+
+ assert derive_requested_output_extraction_plan(flow_evidence=[entry], labels_by_path=labels) is None
+
+ plan = derive_requested_output_extraction_plan(
+ flow_evidence=[entry], labels_by_path=labels, witnessed_by_path={"output.visitors": "8.7K"}
+ )
+ assert plan is not None
+ assert [binding.output_path for binding in plan.live_reads] == ["output.visitors"]
+ assert [binding.relation_label for binding in plan.live_reads] == ["Visitors"]
+
+
+def test_a_witnessed_value_matching_two_relations_binds_nothing() -> None:
+ labels = {"output.visitors": ("the number of visitors in the last week is returned",)}
+ first = _bail_relation("Visitors", selector=".tile")
+ first["value_text"] = "8.7K"
+ second = _bail_relation("Sessions", selector=".other")
+ second["value_text"] = "8.7K"
+
+ assert (
+ derive_requested_output_extraction_plan(
+ flow_evidence=[_bail_entry([first, second])],
+ labels_by_path=labels,
+ witnessed_by_path={"output.visitors": "8.7K"},
+ )
+ is None
+ )
+
+
+def test_key_value_bind_survives_a_truncated_container_channel() -> None:
+ # A live dashboard capped its container capture while every tile relation was still captured;
+ # vetoing the whole packet withheld a tile bind whose own channel was complete.
+ labels = {"output.visitors": ("Visitors",)}
+ entry = _bail_entry([_bail_relation("Visitors", selector=".tile-header")])
+ entry["evidence"]["result_containers_truncated"] = True
+
+ plan = derive_requested_output_extraction_plan(flow_evidence=[entry], labels_by_path=labels)
+ assert plan is not None
+ assert [binding.output_path for binding in plan.live_reads] == ["output.visitors"]
+
+ entry["evidence"]["key_value_relations_truncated"] = True
+ assert derive_requested_output_extraction_plan(flow_evidence=[entry], labels_by_path=labels) is None
+
+
+def test_bail_reason_names_the_truncated_channel_behind_an_unbound_path() -> None:
+ labels = {"output.visitors": ("visitors",)}
+ entry = _bail_entry([_bail_relation("Sessions", selector=".other")])
+ entry["evidence"]["result_containers_truncated"] = True
+
+ assert derivation_bail_reason(flow_evidence=[entry], labels_by_path=labels) == (
+ "bindings[output.visitors:witness-not-declared truncated=['result_containers']]"
+ )
+
+
def test_only_exact_configured_aliases_bind_observed_labels() -> None:
fuzzy = _flow_packet()
fuzzy["evidence"]["key_value_relations"][0]["key_text"] = "Approximate record value"
@@ -231,6 +438,31 @@ def _packet_with_reveal(*relations: dict) -> dict[str, object]:
return packet
+_TYPED_LABEL_BY_PATH = {"output.visitors_last_7_days": ("visitors",)}
+_OUTCOME_PROSE_BY_PATH = {"output.visitors_last_7_days": ("the number of visitors in the last 7 days is output",)}
+_LIVE_COUNTER_ROW = _custody_relation("recently online", "4", selector="#live-counter", child_index=1, child_count=3)
+_CHART_AXIS_ROW = _custody_relation("", "0", selector="#trend-chart", child_index=1, child_count=4)
+_METRIC_TILE_ROW = _custody_relation("Visitors", "8.45K", selector="#visitors-tile", child_index=2, child_count=4)
+
+
+def test_typed_requested_output_label_binds_the_metric_tile_not_the_counter_or_axis() -> None:
+ plan = _derive(
+ packet=_custody_packet(_LIVE_COUNTER_ROW, _CHART_AXIS_ROW, _METRIC_TILE_ROW),
+ labels_by_path=_TYPED_LABEL_BY_PATH,
+ )
+
+ assert plan is not None
+ assert [(binding.output_path, binding.relation_label, binding.selector) for binding in plan.live_reads] == [
+ ("output.visitors_last_7_days", "Visitors", "#visitors-tile")
+ ]
+ assert _derive(packet=_custody_packet(_LIVE_COUNTER_ROW), labels_by_path=_TYPED_LABEL_BY_PATH) is None
+ assert _derive(packet=_custody_packet(_CHART_AXIS_ROW), labels_by_path=_TYPED_LABEL_BY_PATH) is None
+
+
+def test_metric_tile_binds_on_the_typed_label_and_never_on_the_outcome_sentence() -> None:
+ assert _derive(packet=_custody_packet(_METRIC_TILE_ROW), labels_by_path=_OUTCOME_PROSE_BY_PATH) is None
+
+
def test_exact_path_rejects_empty_label_and_binds_configured_label() -> None:
assert _exact_path("", LABELS_BY_PATH) is None
assert _exact_path("Overall State", LABELS_BY_PATH) == "output.overall_state"
@@ -471,3 +703,583 @@ def test_categorical_column_requires_repetition() -> None:
packet["result_containers"][0]["rows"][1]["cells"][1]["text"] = "Started"
bindings = _table_shape_bindings(packet, resolved)
assert [binding.output_path for binding in bindings] == ["output.sites[].depot"]
+
+
+def _bail_entry(
+ relations: list[dict], *, reached_via: str = "current_page", drop: str | tuple[str, ...] | None = None
+) -> dict:
+ evidence = {
+ "source_tool": "inspect_page_for_composition",
+ "result_containers_truncated": False,
+ "key_value_relations_truncated": False,
+ "inspection_warnings": [],
+ "key_value_relations": relations,
+ "result_containers": [],
+ }
+ for key in (drop,) if isinstance(drop, str) else drop or ():
+ evidence.pop(key)
+ return {"step": 4, "reached_via": reached_via, "had_bounded_schema": True, "evidence": evidence}
+
+
+def _bail_relation(key: str, *, selector: str, matches: int = 1, position: int = 0) -> dict:
+ return {
+ "key_text": key,
+ "value_text": "-8.5%",
+ "container_selector": selector,
+ "container_match_count": matches,
+ "container_position": position,
+ "value_child_index": 1,
+ "direct_child_count": 2,
+ "visible": True,
+ "value_visible": True,
+ }
+
+
+def test_bail_reason_names_each_guard_distinctly() -> None:
+ # Ten live runs produced derived=0 with the tile on screen; each cause previously cost a
+ # ~15-minute run to distinguish. The reason string must separate them in one log line.
+ labels = {"output.visitors": ("visitors",)}
+ tile = _bail_relation("Visitors", selector=".tile-header")
+ sidebar = _bail_relation("Visitors", selector=".sidebar-item", matches=3)
+
+ assert derivation_bail_reason(flow_evidence=[], labels_by_path={}) == "no-authoritative-paths"
+ assert derivation_bail_reason(flow_evidence=[], labels_by_path=labels) == "no-bindable-entry"
+ assert (
+ derivation_bail_reason(flow_evidence=[_bail_entry([tile], reached_via="post_run")], labels_by_path=labels)
+ == "no-bindable-entry"
+ )
+ assert derivation_bail_reason(flow_evidence=[_bail_entry([tile])], labels_by_path=labels) == (
+ "table-consistency-or-derived"
+ )
+ assert "ambiguous=['output.visitors(n=2)']" in derivation_bail_reason(
+ flow_evidence=[_bail_entry([tile, sidebar])], labels_by_path=labels
+ )
+ assert "output.visitors:witness-not-declared" in derivation_bail_reason(
+ flow_evidence=[_bail_entry([_bail_relation("Sessions", selector=".other")])], labels_by_path=labels
+ )
+ # Truncation stopped being terminal once a witness could bind through it, so it is reported as
+ # context beside the per-path reason rather than as the answer.
+ truncated_reason = derivation_bail_reason(
+ flow_evidence=[_bail_entry([tile], drop=("result_containers_truncated", "key_value_relations_truncated"))],
+ labels_by_path=labels,
+ )
+ assert "output.visitors:witness-not-declared" in truncated_reason
+ assert "key_value_relations" in truncated_reason
+ assert derivation_bail_reason(
+ flow_evidence=[_bail_entry([tile], reached_via="interaction")], labels_by_path=labels
+ ).startswith("packet-source-tool[interaction:")
+
+
+def _truncated_entry(relations: list[dict]) -> dict:
+ return {
+ "step": 1,
+ "reached_via": "current_page",
+ "had_bounded_schema": True,
+ "evidence": {
+ "source_tool": "inspect_page_for_composition",
+ "inspection_warnings": [],
+ "key_value_relations": relations,
+ "result_containers": [],
+ "key_value_relations_truncated": True,
+ "result_containers_truncated": True,
+ },
+ }
+
+
+def _counted_relation(key: str, value: str, selector: str, page_count: int) -> dict:
+ relation = _bail_relation(key, selector=selector)
+ relation["value_text"] = value
+ relation["key_text_walked_count"] = page_count
+ return relation
+
+
+def test_a_label_counted_once_page_wide_binds_from_a_truncated_channel() -> None:
+ # A real dashboard trips both capture caps, so every requested output was refused before any
+ # binding was attempted, even though the label it needed was captured and unique.
+ unique = _counted_relation("Visitors", "8.7K", ".tile", 1)
+
+ plan = derive_requested_output_extraction_plan(
+ flow_evidence=[_truncated_entry([unique])], labels_by_path={"output.visitors": ("Visitors",)}
+ )
+
+ assert plan is not None
+ assert [binding.output_path for binding in plan.live_reads] == ["output.visitors"]
+
+
+def test_a_label_the_page_repeats_beyond_the_capture_still_refuses() -> None:
+ repeated = _counted_relation("agents", "v0", ".row", 34)
+
+ assert (
+ derive_requested_output_extraction_plan(
+ flow_evidence=[_truncated_entry([repeated])], labels_by_path={"output.agents": ("agents",)}
+ )
+ is None
+ )
+
+
+def test_two_aliases_for_one_path_refuse_even_when_each_is_counted_once() -> None:
+ # Each alias occurring once still yields two bindings for the path, so the count alone is not
+ # licence to bind.
+ first = _counted_relation("Visitors", "8.7K", ".tile", 1)
+ second = _counted_relation("Unique visitors", "8.7K", ".other", 1)
+
+ assert (
+ derive_requested_output_extraction_plan(
+ flow_evidence=[_truncated_entry([first, second])],
+ labels_by_path={"output.visitors": ("Visitors", "Unique visitors")},
+ )
+ is None
+ )
+
+
+def test_one_relation_two_paths_name_refuses_rather_than_reporting_one_tile_twice() -> None:
+ # Counting is per path, so without a cross-path check both paths bind the single "Total" tile and
+ # the block reports one number as two different outputs.
+ only = _counted_relation("Total", "8.7K", ".tile", 1)
+
+ assert (
+ derive_requested_output_extraction_plan(
+ flow_evidence=[_truncated_entry([only])],
+ labels_by_path={"output.visitors": ("Total",), "output.sessions": ("Total",)},
+ )
+ is None
+ )
+
+
+def _metric_card(label: str, figure: str, index: int) -> str:
+ # A bare heading rather than a label/delta pair, so nothing here is two-child shaped and the cap
+ # can only be reached through the metric-card pass.
+ return f'<div id="card-{index}"><h3>{label}</h3><div>{figure}</div><div>vs prior</div></div>'
+
+
+def _capped_dashboard_packet() -> dict:
+ # 25 cards past a 24-relation cap: "Visitors" is captured and walked once, "Total" is captured
+ # once but walked twice because its twin sits beyond the cap.
+ cards = [_metric_card("Visitors", "8.83K", 0), _metric_card("Total", "10.7K", 1)]
+ cards += [_metric_card(f"Metric {index}", f"{index}.0K", index + 2) for index in range(22)]
+ cards.append(_metric_card("Total", "42.0K", 24))
+ parsed = parse_composition_html(
+ f"<body><main>{''.join(cards)}</main></body>",
+ inspected_url="https://example.test/web",
+ current_url="https://example.test/web",
+ )
+ return {"step": 1, "reached_via": "current_page", "had_bounded_schema": True, "evidence": parsed}
+
+
+def test_the_extractor_reports_truncation_when_the_cap_drops_a_metric_card() -> None:
+ # A channel that loses relations without saying so reads as complete, and a complete list is the
+ # one thing the binder is entitled to trust.
+ packet = _capped_dashboard_packet()
+
+ assert packet["evidence"]["key_value_relations_truncated"] is True
+
+
+def test_a_walked_once_label_binds_and_its_walked_twice_neighbour_refuses_from_one_real_capture() -> None:
+ packet = _capped_dashboard_packet()
+
+ bound = derive_requested_output_extraction_plan(
+ flow_evidence=[packet], labels_by_path={"output.visitors": ("Visitors",)}
+ )
+ repeated = derive_requested_output_extraction_plan(
+ flow_evidence=[packet], labels_by_path={"output.total": ("Total",)}
+ )
+
+ assert bound is not None
+ assert [binding.relation_label for binding in bound.live_reads] == ["Visitors"]
+ assert repeated is None
+
+
+def test_a_shared_label_does_not_take_down_an_unrelated_path_in_the_same_packet() -> None:
+ shared = _counted_relation("Total", "8.7K", ".tile", 1)
+ distinct = _counted_relation("Visitors", "4.3K", ".other", 1)
+
+ plan = derive_requested_output_extraction_plan(
+ flow_evidence=[_truncated_entry([shared, distinct])],
+ labels_by_path={"output.visitors": ("Visitors",)},
+ )
+
+ assert plan is not None
+ assert [binding.relation_label for binding in plan.live_reads] == ["Visitors"]
+
+
+def test_a_page_that_never_uses_the_requested_wording_still_offers_what_it_shows() -> None:
+ # Live shape (SKY-13226): the request asked for "azure errors" while the page labelled the count
+ # "logs found", so exact label binding could never join them however plainly the value was shown.
+ packet = _truncated_entry(
+ [
+ _counted_relation("logs found", "1.22K", ".count", 1),
+ _counted_relation("76.1%", "of total errors", ".pct", 1),
+ ]
+ )
+
+ assert unbound_candidate_relations([packet]) == [("logs found", "1.22K"), ("76.1%", "of total errors")]
+
+
+def test_an_unreadable_relation_is_not_offered_as_a_candidate() -> None:
+ hidden = dict(_counted_relation("logs found", "1.22K", ".count", 1), visible=False)
+
+ assert unbound_candidate_relations([_truncated_entry([hidden])]) == []
+
+
+def test_a_dialog_only_capture_does_not_outrank_the_page_it_covered() -> None:
+ # Live shape (SKY-13226): a time-zone dialog left the capture on the log page describing only its
+ # own two buttons. Three walks each choose "the freshest bindable packet", so excluding it from
+ # one still let the other two select it; driven through the shared append seam because the
+ # discriminator alone proves nothing about which packet a walk then picks.
+ ctx = SimpleNamespace(flow_evidence=[])
+ dismiss = "No, keep the current setting"
+ page = _truncated_entry([_counted_relation("logs found", "1.31K", ".count", 1)])["evidence"]
+ dialog = dict(
+ page,
+ key_value_relations=[_counted_relation(dismiss, "Yes, switch", ".dialog", 1)],
+ modal_overlays=[{"dismiss_controls": [{"text": dismiss}, {"text": "Yes, switch"}]}],
+ )
+
+ _append_flow_evidence(ctx, page, reached_via="current_page")
+ _append_flow_evidence(ctx, dialog, reached_via="current_page")
+
+ assert ctx.flow_evidence[1]["obstructed"] is True
+ assert unbound_candidate_relations(ctx.flow_evidence) == [("logs found", "1.31K")]
+ assert bindable_candidate_headings(ctx.flow_evidence) == ["logs found"]
+
+
+def test_a_live_counter_binds_through_the_packet_its_witness_came_from() -> None:
+ # Live shape (SKY-13226): the page recounts between the read and the next capture, so the freshest
+ # packet shows 1.42K while the read witnessed 1.41K, and a join on the freshest packet alone can
+ # never succeed for a moving figure. The witnessed value names its own contemporaneous packet.
+ def _intact(entry: dict, step: int) -> dict:
+ entry["evidence"] = dict(entry["evidence"], key_value_relations_truncated=False)
+ return dict(entry, step=step)
+
+ at_read = _intact(_truncated_entry([_counted_relation("logs found", "1.41K", ".count", 1)]), 1)
+ ticked = _intact(_truncated_entry([_counted_relation("logs found", "1.42K", ".count", 1)]), 2)
+
+ plan = derive_requested_output_extraction_plan(
+ flow_evidence=[at_read, ticked],
+ labels_by_path={"output.azure_error_count": ("azure",)},
+ witnessed_by_path={"output.azure_error_count": "1.41K"},
+ )
+
+ assert plan is not None
+ assert [b.relation_label for b in plan.live_reads] == ["logs found"]
+ assert plan.observation_step == at_read["step"]
+
+
+def test_without_a_witness_only_the_freshest_packet_is_tried() -> None:
+ stale = _truncated_entry([_counted_relation("azure", "7", ".tile", 1)])
+ fresh = dict(_truncated_entry([_counted_relation("unrelated", "9", ".other", 1)]), step=2)
+
+ assert (
+ derive_requested_output_extraction_plan(
+ flow_evidence=[stale, fresh], labels_by_path={"output.azure_error_count": ("azure",)}
+ )
+ is None
+ )
+
+
+# --- witness admission matrix (SKY-13226) -------------------------------------------------------
+#
+# The value witness binds on the value a read observed, so it is the route for a page whose wording
+# shares nothing with the request. These pin what admits such a binding and what still refuses it.
+
+
+def _witness_relation(key: str, value: str, *, selector: str, child: int, count: int, walked: int = 1) -> dict:
+ return {
+ "key_text": key,
+ "value_text": value,
+ "container_selector": selector,
+ "container_match_count": 1,
+ "container_position": 0,
+ "value_child_index": child,
+ "direct_child_count": count,
+ "value_text_walked_count": walked,
+ "visible": True,
+ "value_visible": True,
+ }
+
+
+def _witness_entry(relations: list[dict], *, truncated: bool = False) -> dict:
+ return {
+ "step": 1,
+ "reached_via": "current_page",
+ "had_bounded_schema": True,
+ "evidence": {
+ "source_tool": "inspect_page_for_composition",
+ "inspection_warnings": [],
+ "key_value_relations": relations,
+ "result_containers": [],
+ "key_value_relations_truncated": truncated,
+ "result_containers_truncated": False,
+ },
+ }
+
+
+def test_a_witness_binds_a_page_whose_wording_the_request_never_uses() -> None:
+ # Live shape (SKY-13226): the page labels the requested quantity "logs found" while the request
+ # says "azure errors", so no label can join them however plainly the value is shown.
+ entry = _witness_entry([_witness_relation("logs found", "1.41K", selector=".count", child=0, count=2)])
+
+ plan = derive_requested_output_extraction_plan(
+ flow_evidence=[entry],
+ labels_by_path={},
+ witnessed_by_path={"output.azure_error_count": "1.41K"},
+ requested_paths={"output.azure_error_count"},
+ )
+
+ assert plan is not None
+ assert [(b.relation_label, b.child_index, b.child_count) for b in plan.live_reads] == [("logs found", 0, 2)]
+ # Goal completion checks the requested paths are a subset of this; built from the labels it was
+ # empty here, so a label-free bind produced a plan the turn could never call complete.
+ assert plan.requested_output_paths == ("output.azure_error_count",)
+
+
+def test_a_designated_value_binds_the_tile_no_label_and_no_scalar_read_could_reach() -> None:
+ # Live shape (SKY-13226): the page labels the tile "Visitors" while the request calls it
+ # "the number of visitors in the last week", and the model's own read returned the label and
+ # the delta chip without the figure. The value the page confirmed is what joins them.
+ entry = _witness_entry([_witness_relation("Visitors", "7.82K", selector=".card", child=1, count=3)])
+
+ assert (
+ derive_requested_output_extraction_plan(
+ flow_evidence=[entry],
+ labels_by_path={"output.visitor_count": ("the number of visitors in the last week",)},
+ witnessed_by_path={"output.visitor_count": "Visitors\n-18.0%"},
+ requested_paths={"output.visitor_count"},
+ )
+ is None
+ )
+
+ designated = derive_requested_output_extraction_plan(
+ flow_evidence=[entry],
+ labels_by_path={},
+ witnessed_by_path={"output.visitor_count": "7.82K"},
+ requested_paths={"output.visitor_count"},
+ )
+
+ assert designated is not None
+ binding = designated.live_reads[0]
+ # Bound through the relation channel, so the generated read keeps its runtime heading proof
+ # rather than pinning the leaf directly.
+ assert (binding.relation_label, binding.child_index, binding.child_count) == ("Visitors", 1, 3)
+
+
+def test_a_designation_pins_the_element_when_no_packet_relation_carries_it() -> None:
+ plan = plan_from_designations(
+ [{"output_path": "output.total", "selector": ".val", "match_count": 1, "position": 0, "text": "7.82K"}],
+ {"output.total"},
+ )
+
+ assert plan is not None
+ assert plan.requested_output_paths == ("output.total",)
+ binding = plan.live_reads[0]
+ # child_count 0 is what tells synthesis to read the element itself rather than walk a child slot.
+ assert (binding.selector, binding.selector_count, binding.child_count) == (".val", 1, 0)
+
+
+def test_a_designation_the_page_saw_but_could_not_pin_still_witnesses_its_value() -> None:
+ # Live shape (SKY-13226): the log explorer prints "1.44K" on three visible leaves, so the probe
+ # cannot resolve one element and rejects. The page still shows that value in exactly one
+ # relation, and the model naming it is what the witness needs.
+ entry = _witness_entry([_witness_relation("logs found", "1.44K", selector=".count", child=0, count=2)])
+ value_only = [{"output_path": "output.azure_error_count", "selector": "", "text": "1.44K"}]
+
+ # It pins nothing on its own — there is no element to read.
+ assert plan_from_designations(value_only, {"output.azure_error_count"}) is None
+
+ plan = derive_requested_output_extraction_plan(
+ flow_evidence=[entry],
+ labels_by_path={},
+ witnessed_by_path={d["output_path"]: d["text"] for d in value_only},
+ requested_paths={"output.azure_error_count"},
+ )
+
+ assert plan is not None
+ binding = plan.live_reads[0]
+ assert (binding.relation_label, binding.child_index, binding.child_count) == ("logs found", 0, 2)
+
+
+def test_a_value_only_designation_leaves_the_path_uncovered_rather_than_half_pinned() -> None:
+ pinned_a = {"output_path": "output.a", "selector": ".a", "match_count": 1, "position": 0, "text": "1"}
+ pinned_b = {"output_path": "output.b", "selector": ".b", "match_count": 1, "position": 0, "text": "2"}
+ value_only_b = {"output_path": "output.b", "selector": "", "text": "2"}
+ scope = {"output.a", "output.b"}
+
+ # Withheld, so the uncovered path falls through to the witness channel rather than shipping a
+ # plan that reads one of the two requested values.
+ assert plan_from_designations([pinned_a, value_only_b], scope) is None
+ # The value-only entry is skipped, not fatal: pinning both still builds the plan.
+ plan = plan_from_designations([pinned_a, pinned_b], scope)
+ assert plan is not None
+ assert [b.output_path for b in plan.live_reads] == ["output.a", "output.b"]
+
+
+def test_a_designation_the_page_could_not_pin_uniquely_binds_nothing() -> None:
+ assert (
+ plan_from_designations(
+ [{"output_path": "output.total", "selector": ".val", "match_count": 1, "position": 3}],
+ {"output.total"},
+ )
+ is None
+ )
+ assert (
+ plan_from_designations(
+ [{"output_path": "output.other", "selector": ".val", "match_count": 1, "position": 0}],
+ {"output.total"},
+ )
+ is None
+ )
+
+
+def test_a_witness_binds_through_a_truncated_channel_when_its_value_is_page_unique() -> None:
+ # A dashboard always trips the relation cap, so requiring an intact channel switched the witness
+ # off for every page rich enough to need it. Page-wide uniqueness of the value is the proof.
+ entry = _witness_entry(
+ [_witness_relation("Visitors", "7.82K", selector=".tile", child=1, count=3, walked=1)], truncated=True
+ )
+
+ plan = derive_requested_output_extraction_plan(
+ flow_evidence=[entry],
+ labels_by_path={"output.visitors": ("the number of visitors is returned",)},
+ witnessed_by_path={"output.visitors": "7.82K"},
+ requested_paths={"output.visitors"},
+ )
+
+ assert plan is not None
+ assert [(b.relation_label, b.child_index, b.child_count) for b in plan.live_reads] == [("Visitors", 1, 3)]
+
+
+def test_a_truncated_channel_refuses_a_witness_whose_page_count_is_unproved() -> None:
+ entry = _witness_entry(
+ [
+ dict(
+ _witness_relation("Visitors", "7.82K", selector=".tile", child=1, count=3), value_text_walked_count=None
+ )
+ ],
+ truncated=True,
+ )
+
+ assert (
+ derive_requested_output_extraction_plan(
+ flow_evidence=[entry],
+ labels_by_path={},
+ witnessed_by_path={"output.visitors": "7.82K"},
+ requested_paths={"output.visitors"},
+ )
+ is None
+ )
+
+
+def test_a_truncated_channel_refuses_a_witness_the_page_shows_twice() -> None:
+ entry = _witness_entry(
+ [_witness_relation("Visitors", "7.82K", selector=".tile", child=1, count=3, walked=2)], truncated=True
+ )
+
+ assert (
+ derive_requested_output_extraction_plan(
+ flow_evidence=[entry],
+ labels_by_path={},
+ witnessed_by_path={"output.visitors": "7.82K"},
+ requested_paths={"output.visitors"},
+ )
+ is None
+ )
+
+
+def test_one_relation_cannot_answer_two_requested_outputs() -> None:
+ entry = _witness_entry([_witness_relation("Visitors", "7.82K", selector=".tile", child=1, count=3)])
+
+ assert (
+ derive_requested_output_extraction_plan(
+ flow_evidence=[entry],
+ labels_by_path={},
+ witnessed_by_path={"output.a": "7.82K", "output.b": "7.82K"},
+ requested_paths={"output.a", "output.b"},
+ )
+ is None
+ )
+
+
+def test_a_requested_path_left_unwitnessed_withholds_the_whole_plan() -> None:
+ entry = _witness_entry(
+ [
+ _witness_relation("Visitors", "7.82K", selector=".a", child=1, count=3),
+ _witness_relation("Sessions", "9.39K", selector=".b", child=1, count=3),
+ ]
+ )
+
+ assert (
+ derive_requested_output_extraction_plan(
+ flow_evidence=[entry],
+ labels_by_path={},
+ witnessed_by_path={"output.visitors": "7.82K"},
+ requested_paths={"output.visitors", "output.sessions"},
+ )
+ is None
+ )
+
+
+def test_a_witness_binds_values_that_are_not_quantities() -> None:
+ # Extraction is not always a number, and the join is exact-value equality, so an identifier, an
+ # address and a date bind on the same evidence a figure would.
+ entry = _witness_entry(
+ [
+ _witness_relation("Order", "ORD-88213", selector=".o", child=1, count=2),
+ _witness_relation("Ship to", "12 Harbor Way, Portsmouth", selector=".s", child=1, count=2),
+ _witness_relation("Placed", "2026-07-30", selector=".p", child=1, count=2),
+ ]
+ )
+
+ plan = derive_requested_output_extraction_plan(
+ flow_evidence=[entry],
+ labels_by_path={},
+ witnessed_by_path={"output.order_id": "ORD-88213"},
+ requested_paths={"output.order_id"},
+ )
+
+ assert plan is not None
+ assert [b.relation_label for b in plan.live_reads] == ["Order"]
+
+
+def test_a_value_repeated_across_rows_witnesses_nothing() -> None:
+ entry = _witness_entry(
+ [
+ _witness_relation("Row 1 status", "Shipped", selector=".r1", child=1, count=2),
+ _witness_relation("Row 2 status", "Shipped", selector=".r2", child=1, count=2),
+ ]
+ )
+
+ assert (
+ derive_requested_output_extraction_plan(
+ flow_evidence=[entry],
+ labels_by_path={},
+ witnessed_by_path={"output.status": "Shipped"},
+ requested_paths={"output.status"},
+ )
+ is None
+ )
+
+
+def test_bail_reason_separates_why_each_witness_channel_could_not_answer() -> None:
+ # A vocabulary miss, an undeclared read and a value the page shows twice want three different
+ # fixes, and each previously read as the same "no-labels" (SKY-13226).
+ shown_twice = [
+ _bail_relation("Row 1", selector=".r1"),
+ _bail_relation("Row 2", selector=".r2"),
+ ]
+ for relation in shown_twice:
+ relation["value_text"] = "Shipped"
+ entry = _bail_entry(shown_twice)
+ paths = {"output.status"}
+
+ assert "witness-not-declared" in derivation_bail_reason(
+ flow_evidence=[entry], labels_by_path={}, requested_paths=paths
+ )
+ assert "witness-not-present" in derivation_bail_reason(
+ flow_evidence=[entry],
+ labels_by_path={},
+ witnessed_by_path={"output.status": "Delivered"},
+ requested_paths=paths,
+ )
+ assert "witness-ambiguous[n=2]" in derivation_bail_reason(
+ flow_evidence=[entry], labels_by_path={}, witnessed_by_path={"output.status": "Shipped"}, requested_paths=paths
+ )
diff --git a/tests/unit/test_copilot_positive_evidence_carriers.py b/tests/unit/test_copilot_positive_evidence_carriers.py
index 5126d37b5..1ea67be52 100644
--- a/tests/unit/test_copilot_positive_evidence_carriers.py
+++ b/tests/unit/test_copilot_positive_evidence_carriers.py
@@ -931,3 +931,48 @@ def test_value_present_abstains_when_registered_output_producer_block_failed() -
)
verdict = grade_requested_output_criteria(ctx, [_value_present_criterion()], snapshot)[0]
assert verdict.state != "satisfied"
+
+
+# Byte-exact rows from the SKY-13332 live witness (wr_557702915819208430) and the SKY-13200 custody
+# dir: a dashboard renders "8.45K" while the block registers the integer 8450, and the same tile's
+# "-8.0%" delta yields a bare 8 that must not pass as the count.
+_DELTA_DIGIT_EVIDENCE = "Visitors\n-8.0%"
+_TILE_EVIDENCE = "Visitors\n-15.0%\n8.45K\nvs. 9.99K prior"
+
+
+def test_literal_containment_alone_would_miss_the_correct_extraction() -> None:
+ # Pins why the numeric-equivalence half is load-bearing: the correct value is not a substring of
+ # the tile that displays it, so a containment-only rule leaves a right answer unconfirmable.
+ assert _boundary_delimited_present("8450", _TILE_EVIDENCE) is False
+
+
+def _exact_value_criterion(value: str) -> object:
+ return make_completion_criterion(
+ "c_star",
+ "the number of stars for https://example.com/example-org/example-repo is retrieved",
+ output_path="output.star_count",
+ expected_output_value=value,
+ )
+
+
+def test_registered_scalar_does_not_witness_itself() -> None:
+ # The delta digit reached a verified terminal because the block's own registered output counted
+ # as independent confirmation of that same value.
+ snapshot = RunEvidenceSnapshot(
+ block_outputs={
+ "extract_star_count_output": {
+ "evidence_text": _DELTA_DIGIT_EVIDENCE,
+ "output": {"star_count": 8},
+ }
+ },
+ block_output_sources={"extract_star_count_output": "registered_output_parameter"},
+ )
+ verdict = grade_requested_output_criteria(_star_ctx(), [_exact_value_criterion("8")], snapshot)[0]
+ assert verdict.state != "satisfied"
+
+
+def test_runtime_output_scalar_still_credits_normally() -> None:
+ # The narrowing targets the self-witnessing source only; ordinary runtime_output grading is
+ # untouched, so the reject does not swallow the legitimate path.
+ verdict = grade_requested_output_criteria(_star_ctx(), [_exact_value_criterion("22600")], _star_snapshot())[0]
+ assert verdict.state == "satisfied"
diff --git a/tests/unit/test_copilot_request_policy_completion_criteria.py b/tests/unit/test_copilot_request_policy_completion_criteria.py
index ee7ea3ae9..2a642423e 100644
--- a/tests/unit/test_copilot_request_policy_completion_criteria.py
+++ b/tests/unit/test_copilot_request_policy_completion_criteria.py
@@ -29,6 +29,7 @@ from skyvern.forge.sdk.copilot.request_policy import (
RequestPolicy,
_apply_classifier_typed_requested_output_corroborators,
_apply_requested_output_completion_criteria,
+ _apply_requested_output_labels,
_apply_validation_classification_completion_criteria,
_classifier_fallback_policy,
_criterion_grounding_mode,
@@ -2834,3 +2835,104 @@ def test_to_trace_data_reports_mint_degraded_criteria_without_output_path() -> N
assert data["mint_degraded_criterion_count"] == 1
assert data["mint_degraded_criterion_0_id"] == "c0"
assert data["mint_degraded_criterion_0_mint_degrade"] == "contingent_missing_antecedent"
+
+
+def test_classifier_authored_requested_output_criterion_carries_short_typed_label() -> None:
+ message = "Go to the analytics dashboard and output the number of visitors in the last 7 days"
+ policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="c0",
+ outcome="the number of visitors in the last 7 days is output",
+ level="run",
+ output_path="output.visitors_last_7_days",
+ )
+ ]
+ )
+
+ _apply_requested_output_labels(policy, message)
+
+ assert [criterion.requested_output_label for criterion in policy.completion_criteria] == ["visitors"]
+
+
+def test_canonical_requested_output_mint_carries_the_label_it_renders_into_prose() -> None:
+ message = "Go to the analytics dashboard and output the number of visitors in the last 7 days"
+ policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="c0",
+ outcome="the number of visitors in the last 7 days is output",
+ level="run",
+ output_path="output.visitors_last_7_days",
+ )
+ ]
+ )
+
+ _apply_requested_output_completion_criteria(policy, message)
+
+ minted = [criterion for criterion in policy.completion_criteria if criterion.output_path]
+ assert [criterion.requested_output_label for criterion in minted] == ["visitors"]
+
+
+_LIVE_ANALYTICS_MESSAGE = (
+ "Go to https://analytics.example.test/project/1234/web and output the number of visitors in the last 7 days"
+)
+
+
+def test_label_lands_on_a_classifier_criterion_whose_path_names_the_field_differently() -> None:
+ # Live gap: the detector derives output.visitors while the classifier minted
+ # output.visitors_last_7_days for the same field, so an exact-path join drops the label and the
+ # extraction binder is handed the outcome sentence, which never matches a tile heading.
+ policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="c_visitors",
+ outcome="The returned record includes the number of visitors in the last 7 days.",
+ level="run",
+ output_path="output.visitors_last_7_days",
+ )
+ ]
+ )
+
+ _apply_requested_output_labels(policy, _LIVE_ANALYTICS_MESSAGE)
+
+ (criterion,) = policy.completion_criteria
+ assert criterion.requested_output_label == "visitors"
+
+
+def test_label_lands_via_text_coverage_when_the_alias_join_misses() -> None:
+ # Live W11: the classifier named the path output.visitor_count while the detector derived
+ # output.visitors; the alias join missed and the binder was handed the outcome sentence, which
+ # can never match a page heading.
+ policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="c_visitors",
+ outcome="The number of visitors in the last 7 days is output.",
+ level="run",
+ output_path="output.visitor_count",
+ )
+ ]
+ )
+
+ _apply_requested_output_labels(policy, _LIVE_ANALYTICS_MESSAGE)
+
+ (criterion,) = policy.completion_criteria
+ assert criterion.requested_output_label == "visitors"
+
+
+def test_text_coverage_fallback_stays_silent_when_two_criteria_cover_the_field() -> None:
+ duplicated = [
+ CompletionCriterion(
+ id=f"c_{index}",
+ outcome="The number of visitors in the last 7 days is output.",
+ level="run",
+ output_path=f"output.visitor_count_{index}",
+ )
+ for index in (1, 2)
+ ]
+ policy = RequestPolicy(completion_criteria=duplicated)
+
+ _apply_requested_output_labels(policy, _LIVE_ANALYTICS_MESSAGE)
+
+ assert [criterion.requested_output_label for criterion in policy.completion_criteria] == [None, None]
diff --git a/tests/unit/test_copilot_request_policy_pinability_consumer.py b/tests/unit/test_copilot_request_policy_pinability_consumer.py
index 9f0765953..b96207c83 100644
--- a/tests/unit/test_copilot_request_policy_pinability_consumer.py
+++ b/tests/unit/test_copilot_request_policy_pinability_consumer.py
@@ -42,6 +42,7 @@ from skyvern.forge.sdk.copilot.request_slots import PROMPT_NAME as REQUEST_SLOT_
from skyvern.forge.sdk.copilot.request_slots import (
CanonicalRequestSlotV1,
RequestSlotAntecedentFamily,
+ RequestSlotCandidateRefutationV1,
RequestSlotDatumBindingDeclarationV1,
RequestSlotDatumDeclineDeclarationV1,
RequestSlotDatumTargetV1,
@@ -51,6 +52,7 @@ from skyvern.forge.sdk.copilot.request_slots import (
RequestSlotPlane,
RequestSlotProducerInputV1,
canonicalize_request_slots,
+ request_slot_contracts_agree,
)
from skyvern.forge.sdk.schemas.workflow_copilot import (
WorkflowCopilotChatHistoryMessage,
@@ -328,6 +330,11 @@ def _anchor_only_envelope(
)
for source_quote in ("confirmation number", "account number", "selected start date", "deposit amount")
),
+ # The recorded anchor-only shape predates the candidate obligation; the omitted requested
+ # field must now be answered, and a refutation is that answer for this fixture.
+ candidate_refutations=(
+ RequestSlotCandidateRefutationV1(candidate="next owner", reason="anchor-only recorded shape omitted it"),
+ ),
)
@@ -361,6 +368,7 @@ async def test_independent_request_slot_producer_recovers_primary_classifier_omi
antecedent_family=RequestSlotAntecedentFamily.BLOCKER,
),
),
+ candidate_refutations=_water_refutations(),
)
calls: list[str] = []
@@ -447,6 +455,23 @@ def _semantically_identical_anchor_correction() -> dict[str, Any]:
return corrected
+_WATER_SERVICE_CANDIDATES = (
+ "confirmation number",
+ "account number",
+ "selected start date",
+ "deposit amount",
+ "next owner",
+)
+
+
+def _water_refutations(*covered: str) -> tuple[RequestSlotCandidateRefutationV1, ...]:
+ return tuple(
+ RequestSlotCandidateRefutationV1(candidate=candidate, reason="recorded shape omitted it")
+ for candidate in _WATER_SERVICE_CANDIDATES
+ if candidate not in covered
+ )
+
+
def _example_water_service_envelope() -> RequestSlotEnvelopeV1:
return RequestSlotEnvelopeV1(
version="1",
@@ -1418,7 +1443,11 @@ async def test_invalid_anchor_only_claim_gets_one_correction() -> None:
]
corrected = _synthetic_anchor_only_payload()
corrected["completion_criteria"] = [corrected["completion_criteria"][0]]
- envelope = RequestSlotEnvelopeV1(version="1", slots=(_anchor_only_envelope().slots[0],))
+ envelope = RequestSlotEnvelopeV1(
+ version="1",
+ slots=(_anchor_only_envelope().slots[0],),
+ candidate_refutations=_water_refutations("confirmation number"),
+ )
async def handler(*, prompt: str, prompt_name: str) -> dict[str, Any]:
calls.append(prompt_name)
@@ -1695,6 +1724,7 @@ async def test_p8_admitted_source_fixture_mints_ordered_decidable_canonical_slot
latest_assistant_turn="",
retained_history=(),
global_context="",
+ detected_output_candidates=request_policy_module._detected_output_candidates(P8_QUICKCONNECT_REQUEST),
datum_targets=tuple(
RequestSlotDatumTargetV1(
criterion_index=index,
@@ -2297,6 +2327,12 @@ async def test_primary_self_asserted_binding_is_replaced_by_two_pass_producer_co
antecedent_family=RequestSlotAntecedentFamily.UNCONDITIONAL,
),
),
+ candidate_refutations=(
+ RequestSlotCandidateRefutationV1(candidate="request id", reason="recorded shape omitted it"),
+ RequestSlotCandidateRefutationV1(candidate="provider-captured address", reason="recorded shape omitted it"),
+ RequestSlotCandidateRefutationV1(candidate="requested date", reason="recorded shape omitted it"),
+ RequestSlotCandidateRefutationV1(candidate="status", reason="recorded shape omitted it"),
+ ),
)
async def handler(*, prompt: str, prompt_name: str) -> dict[str, Any]:
@@ -2924,7 +2960,11 @@ async def test_recorded_confirmation_runs_reach_same_run_plane_terminal() -> Non
corrected = _semantically_identical_anchor_correction()
corrected["completion_criteria"] = corrected["completion_criteria"][2:5]
envelope = _example_water_service_envelope()
- recorded_envelope = RequestSlotEnvelopeV1(version="1", slots=envelope.slots[:3])
+ recorded_envelope = RequestSlotEnvelopeV1(
+ version="1",
+ slots=envelope.slots[:3],
+ candidate_refutations=_water_refutations("confirmation number", "account number", "selected start date"),
+ )
request_policy_calls = 0
async def classify_handler(*, prompt: str, prompt_name: str) -> dict[str, Any]:
@@ -3565,3 +3605,96 @@ def test_requested_output_mint_state_treats_value_present_as_decidable() -> None
# A shapeless criterion with no typed shape still degrades.
assert _requested_output_mint_state(None, None, None) == ("decidable", None)
assert _requested_output_mint_state(None, "status_label", None) == ("degraded", "undecidable_judgment")
+
+
+_CANDIDATE_MESSAGE = "Go to the dashboard and output the number of visitors in the last 7 days."
+
+
+def _candidate_input() -> RequestSlotProducerInputV1:
+ return RequestSlotProducerInputV1(
+ version="1",
+ latest_request=_CANDIDATE_MESSAGE,
+ workflow_context="",
+ earliest_user_turn="",
+ latest_prior_user_turn="",
+ latest_assistant_turn="",
+ retained_history=(),
+ global_context="",
+ detected_output_candidates=("visitors",),
+ )
+
+
+def test_an_unaddressed_detected_candidate_is_reported_and_still_yields_a_contract() -> None:
+ # A producer that stays silent on a detected candidate is worth knowing about, but refusing the
+ # envelope costs the whole slot contract and every criterion in it — a live directory request
+ # lost all of its paths that way, and the only thing proven was the silence.
+ contract = canonicalize_request_slots(
+ request=_candidate_input(),
+ envelope=RequestSlotEnvelopeV1(version="1", slots=()),
+ )
+
+ assert contract.count == 0
+
+
+def test_refuted_candidate_is_a_valid_empty_contract_and_rides_the_contract() -> None:
+ contract = canonicalize_request_slots(
+ request=_candidate_input(),
+ envelope=RequestSlotEnvelopeV1(
+ version="1",
+ slots=(),
+ candidate_refutations=(
+ RequestSlotCandidateRefutationV1(
+ candidate="visitors", reason="describes a page section, not a requested value"
+ ),
+ ),
+ ),
+ )
+ assert contract.count == 0
+ assert [refutation.candidate for refutation in contract.candidate_refutations] == ["visitors"]
+
+
+def test_slot_anchored_on_the_candidate_satisfies_completeness() -> None:
+ contract = canonicalize_request_slots(
+ request=_candidate_input(),
+ envelope=RequestSlotEnvelopeV1(
+ version="1",
+ slots=(
+ RequestSlotDeclarationV1(
+ source_id="u0",
+ source_quote="output the number of visitors in the last 7 days",
+ plane=RequestSlotPlane.RUN,
+ pinability=RequestSlotPinability.SHAPELESS_VALID,
+ antecedent_family=RequestSlotAntecedentFamily.UNDECIDABLE,
+ ),
+ ),
+ ),
+ )
+ assert contract.count == 1
+ assert contract.candidate_refutations == ()
+
+
+def test_refutation_of_an_undetected_candidate_is_invalid() -> None:
+ with pytest.raises(ValueError):
+ canonicalize_request_slots(
+ request=_request_slot_input(_CANDIDATE_MESSAGE),
+ envelope=RequestSlotEnvelopeV1(
+ version="1",
+ slots=(),
+ candidate_refutations=(RequestSlotCandidateRefutationV1(candidate="visitors", reason="not requested"),),
+ ),
+ )
+
+
+def test_contracts_that_differ_only_in_refutations_do_not_agree() -> None:
+ def _contract(reason: str) -> Any:
+ return canonicalize_request_slots(
+ request=_candidate_input(),
+ envelope=RequestSlotEnvelopeV1(
+ version="1",
+ slots=(),
+ candidate_refutations=(RequestSlotCandidateRefutationV1(candidate="visitors", reason=reason),),
+ ),
+ )
+
+ assert request_slot_contracts_agree(_contract("reason one"), _contract("reason one"))
+ assert not request_slot_contracts_agree(_contract("reason one"), _contract("reason two"))
diff --git a/tests/unit/test_copilot_request_slots.py b/tests/unit/test_copilot_request_slots.py
index 55117ffa8..b2dbc0abf 100644
--- a/tests/unit/test_copilot_request_slots.py
+++ b/tests/unit/test_copilot_request_slots.py
@@ -25,6 +25,7 @@ from skyvern.forge.sdk.copilot.request_slots import (
RequestSlotSourceV1,
canonicalize_request_slots,
produce_request_slots,
+ request_slot_contract_disagreements,
request_slot_contracts_agree,
request_slot_source_text,
request_slot_sources,
@@ -1417,3 +1418,34 @@ def test_request_digest_covers_every_datum_target_identity_member_and_order() ->
pair_digest = contract_for((base_target, second_target)).request_digest
assert pair_digest != base_digest
assert contract_for((second_target, base_target)).request_digest != pair_digest
+
+
+def test_disagreeing_attempts_name_the_field_that_moved() -> None:
+ request = _input(latest_request="Return status.")
+ pinned = canonicalize_request_slots(
+ request=request,
+ envelope=_envelope(request, ("status", "definition", "pinned")),
+ )
+ shapeless = canonicalize_request_slots(
+ request=request,
+ envelope=_envelope(request, ("status", "run", "shapeless_valid")),
+ )
+
+ # Which judgment flipped is what separates a producer wobbling on one axis from one drifting
+ # wholesale, and a degraded turn records only that it degraded.
+ assert request_slot_contract_disagreements(pinned, shapeless) == ("slots.0.plane", "slots.0.pinability")
+ assert request_slot_contract_disagreements(pinned, pinned) == ()
+ assert not request_slot_contracts_agree(pinned, shapeless)
+ assert request_slot_contracts_agree(pinned, pinned)
+
+
+def test_attempts_differing_in_slot_count_report_the_count_not_a_zip_error() -> None:
+ request = _input(latest_request="Return alpha and beta.")
+ one = canonicalize_request_slots(request=request, envelope=_envelope(request, ("alpha", "run", "shapeless_valid")))
+ two = canonicalize_request_slots(
+ request=request,
+ envelope=_envelope(request, ("alpha", "run", "shapeless_valid"), ("beta", "run", "shapeless_valid")),
+ )
+
+ assert request_slot_contract_disagreements(one, two) == ("count", "slots.count")
+ assert not request_slot_contracts_agree(one, two)
diff --git a/tests/unit/test_copilot_scout_act_observe.py b/tests/unit/test_copilot_scout_act_observe.py
index 6d98ca4cb..2c3bbe685 100644
--- a/tests/unit/test_copilot_scout_act_observe.py
+++ b/tests/unit/test_copilot_scout_act_observe.py
@@ -25,6 +25,7 @@ from skyvern.forge.sdk.copilot.composition_evidence import (
_auto_credit_interaction_observation,
has_bounded_page_schema,
has_witnessed_value_content,
+ parse_composition_structured,
)
from skyvern.forge.sdk.copilot.config import CopilotConfig
from skyvern.forge.sdk.copilot.context import (
@@ -1677,6 +1678,78 @@ class TestCredentialInventoryCarry:
assert next_turn.scout_trajectory[-1]["credential_field"] == "username"
+class TestEvaluateCompositionSteerAppendsFlowEvidence:
+ """The steer that tells the model to compose an extraction must also hand the packet it saw to
+ the grounded plan, or the block it writes is unaimed (SKY-13226)."""
+
+ @pytest.mark.asyncio
+ async def test_composition_steer_appends_the_packet_it_steered_on(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ ctx = _ctx()
+ ctx.flow_evidence = []
+ packet = parse_composition_structured(
+ _bounded_extractor_payload(), inspected_url=_LANDING_URL, current_url=_LANDING_URL
+ )
+ monkeypatch.setattr(
+ scouting_module,
+ "_mint_current_loaded_result_source",
+ lambda _ctx, _packet, *, url: SimpleNamespace(result_container_count=8, table_result_container_count=1),
+ )
+ result: dict[str, Any] = {"data": {"elements": [], "text": ""}}
+
+ await scouting_module._maybe_steer_evaluate_to_action(ctx, result, url=_LANDING_URL, page_evidence=packet)
+
+ assert len(ctx.flow_evidence) == 1
+ entry = ctx.flow_evidence[0]
+ assert entry["reached_via"] == "current_page"
+ assert entry["evidence"]["source_tool"] == "inspect_page_for_composition"
+
+ @pytest.mark.asyncio
+ async def test_appended_packet_is_admissible_to_plan_derivation(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ # Appending is only worth anything if derivation accepts the entry; assert the seam, not the
+ # append alone.
+ from skyvern.forge.sdk.copilot.output_extraction_plan import (
+ _BINDABLE_EVIDENCE_TOOLS,
+ _BINDABLE_REACHED_VIA,
+ )
+
+ ctx = _ctx()
+ ctx.flow_evidence = []
+ packet = parse_composition_structured(
+ _bounded_extractor_payload(), inspected_url=_LANDING_URL, current_url=_LANDING_URL
+ )
+ monkeypatch.setattr(
+ scouting_module,
+ "_mint_current_loaded_result_source",
+ lambda _ctx, _packet, *, url: SimpleNamespace(result_container_count=8, table_result_container_count=1),
+ )
+
+ await scouting_module._maybe_steer_evaluate_to_action(
+ ctx, {"data": {"elements": [], "text": ""}}, url=_LANDING_URL, page_evidence=packet
+ )
+
+ entry = ctx.flow_evidence[0]
+ assert entry["reached_via"] in _BINDABLE_REACHED_VIA
+ assert entry["evidence"]["source_tool"] in _BINDABLE_EVIDENCE_TOOLS[entry["reached_via"]]
+
+ @pytest.mark.asyncio
+ async def test_an_observation_that_mints_no_result_is_still_recorded(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ # Live shape (SKY-13226): the log page carrying the requested count minted no loaded result,
+ # so it was steered as actionable and — because appending sat inside the composition branch —
+ # never reached the binder. Recording an observation does not depend on it steering.
+ ctx = _ctx()
+ ctx.flow_evidence = []
+ packet = parse_composition_structured(
+ _bounded_extractor_payload(), inspected_url=_LANDING_URL, current_url=_LANDING_URL
+ )
+ monkeypatch.setattr(scouting_module, "_mint_current_loaded_result_source", lambda _ctx, _packet, *, url: None)
+
+ await scouting_module._maybe_steer_evaluate_to_action(
+ ctx, {"data": {"elements": [], "text": ""}}, url=_LANDING_URL, page_evidence=packet
+ )
+
+ assert [entry["reached_via"] for entry in ctx.flow_evidence] == ["current_page"]
+
+
class TestScoutPageObservationSignal:
def test_password_control_detected_in_forms(self) -> None:
evidence = {
@@ -1848,3 +1921,66 @@ class TestTerminalActionObservationStampSeam:
accessible_name="Find Address",
)
assert ctx.scout_observed_terminal_criterion_ids == set()
+
+
+class TestOverlayOnlyPacketDefersComposition:
+ """Live capture (SKY-13226): a log query's count was on screen, but a time-zone dialog left the
+ packet carrying only that dialog's two buttons, and the round was spent composing from it."""
+
+ def _evidence(self, relations: list[dict[str, object]]) -> dict[str, object]:
+ return {
+ "modal_overlays": [
+ {
+ "dismiss_controls": [
+ {"text": "No, keep it"},
+ {"text": "Yes, update"},
+ ]
+ }
+ ],
+ "key_value_relations": relations,
+ }
+
+ def test_a_packet_carrying_only_the_dialogs_controls_defers(self) -> None:
+ evidence = self._evidence([{"key_text": "No, keep it", "value_text": "Yes, update"}])
+
+ assert scouting_module.packet_describes_a_clearable_overlay(evidence) is True
+
+ def test_a_dialog_beside_readable_page_content_still_composes(self) -> None:
+ evidence = self._evidence(
+ [
+ {"key_text": "No, keep it", "value_text": "Yes, update"},
+ {"key_text": "logs found", "value_text": "1.22K"},
+ ]
+ )
+
+ assert scouting_module.packet_describes_a_clearable_overlay(evidence) is False
+
+ def test_a_page_with_no_dialog_composes(self) -> None:
+ evidence = {"modal_overlays": [], "key_value_relations": [{"key_text": "logs found", "value_text": "1.22K"}]}
+
+ assert scouting_module.packet_describes_a_clearable_overlay(evidence) is False
+
+
+class TestOverlayDeferralIsNotGatedOnLoadedResults:
+ """A packet describing only a dialog has no results to mint, so gating the deferral on having
+ minted some left it dead in the one case it exists for (SKY-13226, live run DDC)."""
+
+ @pytest.mark.asyncio
+ async def test_an_overlay_only_packet_defers_even_when_no_results_are_minted(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ minted: list[str] = []
+
+ def _never_mint(_ctx: Any, _packet: Any, *, url: str) -> None:
+ minted.append(url)
+ return None
+
+ monkeypatch.setattr(scouting_module, "_mint_current_loaded_result_source", _never_mint)
+ evidence = {
+ "modal_overlays": [{"dismiss_controls": [{"text": "No, keep it"}, {"text": "Yes, update"}]}],
+ "key_value_relations": [{"key_text": "No, keep it", "value_text": "Yes, update"}],
+ }
+
+ assert scouting_module.packet_describes_a_clearable_overlay(evidence) is True
+ # The discriminator must not depend on results existing; minting is skipped entirely.
+ assert minted == []
diff --git a/tests/unit/test_copilot_session_factory.py b/tests/unit/test_copilot_session_factory.py
index b50e2895b..46c6c8358 100644
--- a/tests/unit/test_copilot_session_factory.py
+++ b/tests/unit/test_copilot_session_factory.py
@@ -3,13 +3,23 @@
from __future__ import annotations
import json
+from pathlib import Path
from types import SimpleNamespace
from typing import Any
+import pytest
-def _mk_input_data(items: list[Any]) -> Any:
- """Build a fake CallModelData payload with a model_data.input list."""
- return SimpleNamespace(model_data=SimpleNamespace(input=list(items), instructions=None))
+
+def _mk_input_data(items: list[Any], *, instructions: str | None = None, context: Any = None) -> Any:
+ """Build a fake CallModelData payload with a model_data.input list.
+
+ ``CallModelData.context`` is the run context itself (``TContext | None``), not a wrapper around
+ one; a fake that nests it hides an attribute error behind a passing test.
+ """
+ return SimpleNamespace(
+ model_data=SimpleNamespace(input=list(items), instructions=instructions),
+ context=context,
+ )
class TestFirstTurnCompaction:
@@ -137,3 +147,56 @@ class TestSessionInputCallback:
assert len(combined) == 5
assert combined[0] == goal
assert combined[-1] == new[0]
+
+
+class TestModelInputCapture:
+ """COPILOT_DUMP_MODEL_INPUTS records what the model actually receives, so a prompt or
+ tool-schema change can be replayed offline instead of re-run live.
+ """
+
+ def test_capture_is_inert_and_lossless_when_unset(self, tmp_path: Any, monkeypatch: Any) -> None:
+ from skyvern.forge.sdk.copilot.session_factory import copilot_call_model_input_filter
+
+ monkeypatch.delenv("COPILOT_DUMP_MODEL_INPUTS", raising=False)
+ items = [{"role": "user", "content": "build me a workflow"}]
+
+ result = copilot_call_model_input_filter(_mk_input_data(items))
+
+ assert result.input == items
+ assert list(tmp_path.iterdir()) == []
+
+ def test_capture_records_instructions_and_input(self, tmp_path: Any, monkeypatch: Any) -> None:
+ from skyvern.forge.sdk.copilot.session_factory import copilot_call_model_input_filter
+
+ monkeypatch.setenv("COPILOT_DUMP_MODEL_INPUTS", str(tmp_path))
+ items = [
+ {"role": "user", "content": "output the number of azure errors"},
+ {"type": "function_call_output", "call_id": "c1", "output": '{"ok": true}'},
+ ]
+
+ copilot_call_model_input_filter(_mk_input_data(items, instructions="SYSTEM PROMPT"))
+
+ dumps = sorted(tmp_path.glob("call-*.json"))
+ assert len(dumps) == 1
+ payload = json.loads(dumps[0].read_text())
+ assert payload["instructions"] == "SYSTEM PROMPT"
+ assert payload["input"] == items
+ # A context the derivation helper cannot read must not cost the run its model call.
+ assert payload["requested_output_paths"] == []
+
+ def test_capture_records_a_call_whichever_shape_carries_the_context(
+ self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ # A live turn dumped 2 of 34 calls: the copilot's own hand over the context directly, and
+ # reading it as a wrapper lost every one of them to an AttributeError (SKY-13226).
+ from agents.run_context import RunContextWrapper
+
+ from skyvern.forge.sdk.copilot.session_factory import copilot_call_model_input_filter
+
+ monkeypatch.setenv("COPILOT_DUMP_MODEL_INPUTS", str(tmp_path))
+ items = [{"role": "user", "content": "read the visitor count"}]
+
+ copilot_call_model_input_filter(_mk_input_data(items, context=SimpleNamespace()))
+ copilot_call_model_input_filter(_mk_input_data(items, context=RunContextWrapper(context=SimpleNamespace())))
+
+ assert len(sorted(tmp_path.glob("call-*.json"))) == 2
diff --git a/tests/unit/test_copilot_streaming_adapter.py b/tests/unit/test_copilot_streaming_adapter.py
index bc9f81a12..e90efbedc 100644
--- a/tests/unit/test_copilot_streaming_adapter.py
+++ b/tests/unit/test_copilot_streaming_adapter.py
@@ -1754,3 +1754,30 @@ async def test_goal_satisfied_flush_reuses_the_pending_calls_label() -> None:
assert [p.display_label for p in sent] == ['Editing block "Log In"']
assert narrator_state.design_activity[-1]["displayLabel"] == 'Editing block "Log In"'
+
+
+async def test_design_start_narrates_for_a_plain_agent_context() -> None:
+ from skyvern.forge.sdk.copilot.runtime import AgentContext
+ from skyvern.forge.sdk.copilot.streaming_adapter import maybe_emit_design_start
+
+ sent: list[Any] = []
+
+ class _Stream:
+ async def send(self, update: Any) -> None:
+ sent.append(update)
+
+ stream = _Stream()
+ ctx = AgentContext(
+ organization_id="o_1",
+ workflow_id="w_1",
+ workflow_permanent_id="wpid_1",
+ workflow_yaml="",
+ browser_session_id=None,
+ stream=stream, # type: ignore[arg-type]
+ )
+
+ await maybe_emit_design_start(stream, ctx) # type: ignore[arg-type]
+
+ assert len(sent) == 1
+ assert ctx.design_start_emitted is True