Skip to content

Automate browser based workflows with AI

SKY-11855: bind scout-proven held-out inputs into copilot build-test runs + producer-recorded required_input_unbound (#7101)

Andrew Neilsone594525c7ee1
diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.py
index 7eb389f88..5248bd0bd 100644
--- a/skyvern/forge/sdk/copilot/build_test_outcome.py
+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py
@@ -50,6 +50,7 @@ BuildTestOutcomeReasonCode = Literal[
"metadata_reject",
"output_policy_reject",
"scout_act_observe_hollow_after_interaction",
+ "required_input_unbound",
]
_STRUCTURAL_KEY_VERSION = "recorded_build_test_outcome:v1"
@@ -279,6 +280,29 @@ def authored_block_signatures_from_workflow(
return signatures
+def authored_block_parameter_keys_from_workflow(
+ workflow_yaml: str | None,
+ code_artifact_metadata: object = None,
+) -> dict[str, list[str]]:
+ payload = _authored_structure_payload_from_workflow(workflow_yaml, code_artifact_metadata)
+ if payload is None:
+ return {}
+ result: dict[str, list[str]] = {}
+ code_blocks = payload.get("code_blocks")
+ if not isinstance(code_blocks, list):
+ return result
+ for block in code_blocks:
+ if not isinstance(block, Mapping):
+ continue
+ label = _safe_str(block.get("label"))
+ if not label:
+ continue
+ keys = block.get("parameter_keys")
+ if isinstance(keys, list):
+ result[label] = [_safe_str(key) for key in keys if _safe_str(key)]
+ return result
+
+
def _binding_frontier_facet(outcome: RecordedBuildTestOutcome) -> BindingFrontierFacet:
if outcome.reason_code == "scout_act_observe_hollow_after_interaction":
return "unexecuted_submit"
@@ -286,7 +310,11 @@ def _binding_frontier_facet(outcome: RecordedBuildTestOutcome) -> BindingFrontie
return "amend_in_place"
if outcome.missing_requested_output_facts:
return "value_shape"
- if outcome.reason_code in {"sandbox_unresolved_name", "synthesized_parameter_binding_ambiguous"}:
+ if outcome.reason_code in {
+ "sandbox_unresolved_name",
+ "synthesized_parameter_binding_ambiguous",
+ "required_input_unbound",
+ }:
return "amend_in_place"
if outcome.reason_code in {"outcome_not_demonstrated", "no_meaningful_output", "runtime_missing_output_dependency"}:
return "value_shape"
@@ -786,6 +814,48 @@ def recorded_outcome_from_scout_act_observe_hollow(
)
+def _required_input_unbound_identity(
+ failed_block: Mapping[str, object] | None,
+ referenced_unbound_keys: Sequence[str],
+) -> str:
+ return "required_input_unbound:" + _stable_hash(
+ {
+ "source": "required_input_unbound",
+ "referenced_unbound_keys": sorted({str(key) for key in referenced_unbound_keys}),
+ "block_label": _safe_str(failed_block.get("label")) if failed_block is not None else "",
+ "block_status": _safe_str(failed_block.get("status")) if failed_block is not None else "",
+ }
+ )
+
+
+def _required_input_unbound_outcome(
+ failed_block: Mapping[str, object] | None,
+ block_labels: list[str],
+ workflow_run_id: str | None,
+ authored_structure_signature: str | None,
+ referenced_unbound_keys: Sequence[str],
+) -> RecordedBuildTestOutcome:
+ return RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ attempted_block_label=_safe_str(failed_block.get("label")) if failed_block is not None else "",
+ verdict="repairable_failure",
+ reason_code="required_input_unbound",
+ workflow_run_id=workflow_run_id or None,
+ block_labels=block_labels,
+ structural_failure_identity=_required_input_unbound_identity(failed_block, referenced_unbound_keys),
+ authored_structure_signature=authored_structure_signature,
+ observed_evidence_summary=_bounded_text(
+ "unbound required inputs: " + ", ".join(_clean_list(referenced_unbound_keys))
+ ),
+ key_provenance={
+ "structural_failure_identity": (
+ "resolution-seam unbound required parameter keys referenced by the failed block"
+ )
+ },
+ )
+
+
def recorded_outcome_from_run_blocks_result(
result: Mapping[str, object],
*,
@@ -794,12 +864,20 @@ def recorded_outcome_from_run_blocks_result(
completion_verification: CompletionVerificationResult | None = None,
authored_structure_signature: str | None = None,
registered_output_parameter_payloads: Sequence[Mapping[str, object]] | None = None,
+ unbound_required_parameter_keys: Sequence[str] | None = None,
+ block_parameter_keys: Mapping[str, Sequence[str]] | None = None,
) -> RecordedBuildTestOutcome | None:
data = _dict(result.get("data"))
workflow_run_id = _safe_str(data.get("workflow_run_id"))
blocks = _block_dicts(data.get("blocks"))
failed_block = _first_failed_block(blocks)
block_labels = [_safe_str(block.get("label")) for block in blocks if _safe_str(block.get("label"))]
+ referenced_unbound_keys = _referenced_unbound_input_keys(
+ result,
+ failed_block,
+ unbound_required_parameter_keys or [],
+ block_parameter_keys or {},
+ )
page_refs = _page_evidence_refs(page_evidence)
output_refs = _output_evidence_refs(blocks)
verification_identity = _completion_verification_identity(completion_verification)
@@ -856,6 +934,14 @@ def recorded_outcome_from_run_blocks_result(
observed_evidence_summary=recorded_run_outcome.display_reason or "",
key_provenance={"structural_failure_identity": "run outcome was not evaluated"},
)
+ if referenced_unbound_keys:
+ return _required_input_unbound_outcome(
+ failed_block,
+ block_labels,
+ recorded_run_outcome.workflow_run_id or workflow_run_id or None,
+ authored_structure_signature,
+ referenced_unbound_keys,
+ )
structural_identity = verification_identity
evidence_refs = output_refs
if not structural_identity and not page_refs and not evidence_refs:
@@ -897,6 +983,14 @@ def recorded_outcome_from_run_blocks_result(
failure_categories = _failure_category_refs(data.get("failure_categories"))
status = _safe_str(failed_block.get("status")) if failed_block is not None else run_status
runtime_failure_identity = _runtime_failure_identity(failed_block)
+ if referenced_unbound_keys:
+ return _required_input_unbound_outcome(
+ failed_block,
+ block_labels,
+ workflow_run_id or None,
+ authored_structure_signature,
+ referenced_unbound_keys,
+ )
if not (failure_categories or failure_type or runtime_failure_identity or page_refs or output_refs):
return None
structural_identity = (
@@ -1134,6 +1228,22 @@ def _first_failed_block(blocks: Sequence[Mapping[str, object]]) -> Mapping[str,
return None
+def _referenced_unbound_input_keys(
+ result: Mapping[str, object],
+ failed_block: Mapping[str, object] | None,
+ unbound_required_parameter_keys: Sequence[str],
+ block_parameter_keys: Mapping[str, Sequence[str]],
+) -> list[str]:
+ if bool(result.get("ok")) is not False or failed_block is None:
+ return []
+ label = _safe_str(failed_block.get("label"))
+ referenced = block_parameter_keys.get(label) if label else None
+ if not referenced:
+ return []
+ referenced_set = set(referenced)
+ return [key for key in dict.fromkeys(unbound_required_parameter_keys) if key in referenced_set]
+
+
def _runtime_failure_identity(failed_block: Mapping[str, object] | None) -> str:
if failed_block is None:
return ""
diff --git a/skyvern/forge/sdk/copilot/code_block_synthesis.py b/skyvern/forge/sdk/copilot/code_block_synthesis.py
index 51ee7ebe1..fc3b0380a 100644
--- a/skyvern/forge/sdk/copilot/code_block_synthesis.py
+++ b/skyvern/forge/sdk/copilot/code_block_synthesis.py
@@ -156,6 +156,9 @@ class SynthesisDiagnostics:
truncated: bool = False
dropped_interactions: list[dict[str, Any]] = field(default_factory=list)
locator_provenance: list[dict[str, Any]] = field(default_factory=list)
+ # (trajectory enumerate index -> minted type_text parameter key); diagnostics-only, never serialized.
+ # Recovers the key for a typed field whose value was withheld from default_value (typed_value == "").
+ typed_param_bindings: list[tuple[int, str]] = field(default_factory=list)
@dataclass
@@ -907,6 +910,7 @@ def synthesize_code_block(
parameters.append(parameter)
if typed_identity is not None:
typed_param_keys[typed_identity] = param_key
+ diagnostics.typed_param_bindings.append((trajectory_index, param_key))
lines.append(f"{action_indent}await {locator}.fill(str({param_key}))")
append_step(f"Type into {_step_target(interaction)}", "input_text", line_start)
elif tool_name == CREDENTIAL_FILL_TOOL_NAME:
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index b11605323..c08f663cd 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -151,6 +151,9 @@ class ScoutedInteraction(TypedDict):
typed_value: NotRequired[str]
key: NotRequired[str]
typed_length: NotRequired[int]
+ # Raw scout-typed value for run-scoped test binding, gated at capture by should_reject_type_text_value.
+ # Turn-ephemeral; excluded from every persistence path (default_value promotion, typed identity, YAML).
+ raw_typed_value: NotRequired[str]
role: NotRequired[str]
accessible_name: NotRequired[str]
trajectory_index: NotRequired[int]
@@ -348,6 +351,9 @@ class AgentContext:
# Count of times the scout-act download gate rejected a download-intent block this turn. Bounds
# the author->scout->re-author cycle so a genuinely un-scoutable affordance halts honestly.
download_scout_required_rejections: int = 0
+ # Required parameter keys the build-test resolution seam could not bind from a user param,
+ # a non-empty default, or a scout value. Reset per run; read when composing the run outcome.
+ unbound_required_parameter_keys: list[str] = field(default_factory=list)
# Source page of an in-flight scout action, captured before it may navigate away.
pending_scout_source_url: str | None = None
pending_scout_typed_value: str | None = None
diff --git a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
index 5da38403d..65b8dfb2c 100644
--- a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
+++ b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
@@ -673,6 +673,12 @@ async def _type_text_post_hook(
return landing_failure
_mark_pending_browser_interaction_observation(ctx, tool_name="type_text", url=url)
role, accessible_name = await _resolve_scout_role_name(ctx, selector)
+ value_landed = (
+ isinstance(typed_length, int)
+ and typed_length > 0
+ and isinstance(pending_typed_value, str)
+ and len(pending_typed_value) == typed_length
+ )
typed_value = (
safe_typed_default_value(
pending_typed_value,
@@ -680,12 +686,10 @@ async def _type_text_post_hook(
role=role,
accessible_name=accessible_name,
)
- if isinstance(typed_length, int)
- and typed_length > 0
- and isinstance(pending_typed_value, str)
- and len(pending_typed_value) == typed_length
+ if value_landed
else None
)
+ raw_typed_value = pending_typed_value if value_landed and isinstance(pending_typed_value, str) else ""
_record_scouted_interaction(
ctx,
tool_name="type_text",
@@ -693,6 +697,7 @@ async def _type_text_post_hook(
source_url=source_url,
typed_length=typed_length,
typed_value=typed_value or "",
+ raw_typed_value=raw_typed_value,
role=role,
accessible_name=accessible_name,
)
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index 0b98237ef..9c48b68e7 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -27,6 +27,7 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (
RecordedBuildTestOutcome,
RecordedOutcomeGroundingRequirement,
arm_recorded_outcome_grounding_requirement,
+ authored_block_parameter_keys_from_workflow,
authored_structure_signature_from_workflow,
clear_recorded_outcome_grounding_requirement,
latest_recorded_build_test_outcome_repeated,
@@ -41,6 +42,7 @@ from skyvern.forge.sdk.copilot.code_block_security import (
)
from skyvern.forge.sdk.copilot.code_block_synthesis import (
code_contains_credential_fill,
+ synthesize_code_block,
trajectory_has_credential_fill,
)
from skyvern.forge.sdk.copilot.completion_output_grounding import page_evidence_prose_text
@@ -116,12 +118,14 @@ from skyvern.forge.sdk.copilot.turn_halt import (
stash_repair_ceiling_turn_halt,
stash_turn_halt_from_blocker_signal,
)
+from skyvern.forge.sdk.copilot.typed_value_policy import should_reject_type_text_value
from skyvern.forge.sdk.executor.factory import AsyncExecutorFactory
from skyvern.forge.sdk.routes.workflow_copilot import _process_workflow_yaml
from skyvern.forge.sdk.schemas.workflow_copilot import WorkflowCopilotRunOutcomeUpdate, WorkflowCopilotStreamMessageType
from skyvern.forge.sdk.settings_manager import SettingsManager
from skyvern.forge.sdk.utils.pdf_parser import extract_pdf_file
from skyvern.forge.sdk.workflow.models.block import CodeBlock
+from skyvern.forge.sdk.workflow.models.parameter import WorkflowParameter
from skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowRun, WorkflowRunStatus
from skyvern.schemas.workflows import BlockType
from skyvern.webeye.navigation import is_skip_inner_retry_error
@@ -1255,6 +1259,96 @@ async def _capture_and_store_post_run_page(
ctx.composition_page_evidence = None
+def _scout_ephemeral_values(ctx: CopilotContext, workflow_param_keys: set[str]) -> dict[str, str]:
+ trajectory = list(ctx.scout_trajectory or [])
+ if not trajectory:
+ return {}
+ strict = synthesize_code_block(
+ trajectory,
+ strict_selectors=True,
+ reached_download_target=ctx.reached_download_target,
+ )
+ lenient = synthesize_code_block(
+ trajectory,
+ strict_selectors=False,
+ reached_download_target=ctx.reached_download_target,
+ )
+ if strict is None or lenient is None:
+ return {}
+ lenient_by_index = dict(lenient.diagnostics.typed_param_bindings)
+ agreed_bindings = [
+ (index, key) for index, key in strict.diagnostics.typed_param_bindings if lenient_by_index.get(index) == key
+ ]
+ raw_by_index: dict[int, str] = {}
+ for index, interaction in enumerate(trajectory):
+ raw = interaction.get("raw_typed_value")
+ if not isinstance(raw, str) or not raw:
+ continue
+ selector = interaction.get("selector")
+ intent = " ".join(
+ part for part in (interaction.get("role"), interaction.get("accessible_name")) if isinstance(part, str)
+ )
+ if should_reject_type_text_value(
+ value=raw,
+ selector=selector if isinstance(selector, str) else "",
+ intent=intent,
+ ):
+ continue
+ raw_by_index[index] = raw
+ key_to_values: dict[str, set[str]] = {}
+ value_to_keys: dict[str, set[str]] = {}
+ for index, key in agreed_bindings:
+ raw = raw_by_index.get(index)
+ if not raw or key not in workflow_param_keys:
+ continue
+ key_to_values.setdefault(key, set()).add(raw)
+ value_to_keys.setdefault(raw, set()).add(key)
+ resolved: dict[str, str] = {}
+ for key, values in key_to_values.items():
+ if len(values) != 1:
+ continue
+ raw = next(iter(values))
+ if len(value_to_keys.get(raw, set())) != 1:
+ continue
+ resolved[key] = raw
+ return resolved
+
+
+def _resolve_run_data_and_unbound_keys(
+ all_workflow_params: Sequence[WorkflowParameter],
+ user_params: Mapping[str, Any],
+ scout_ephemeral_values: Mapping[str, str],
+) -> tuple[dict[str, Any], list[str]]:
+ data: dict[str, Any] = {}
+ unbound: list[str] = []
+ for wp in all_workflow_params:
+ if wp.key in user_params:
+ data[wp.key] = user_params[wp.key]
+ continue
+ if wp.default_value is not None and wp.default_value != "":
+ data[wp.key] = wp.default_value
+ continue
+ scout_value = scout_ephemeral_values.get(wp.key)
+ if scout_value:
+ data[wp.key] = scout_value
+ LOG.info(
+ "Bound run-scoped scout value for copilot test run",
+ parameter_key=wp.key,
+ value_length=len(scout_value),
+ )
+ continue
+ placeholder = _placeholder_for_parameter_type(wp.workflow_parameter_type)
+ if placeholder is not None:
+ data[wp.key] = placeholder
+ LOG.info(
+ "Auto-filled missing workflow parameter for copilot test run",
+ parameter_key=wp.key,
+ parameter_type=str(wp.workflow_parameter_type),
+ )
+ unbound.append(wp.key)
+ return data, unbound
+
+
async def _run_blocks_and_collect_debug(
params: dict[str, Any],
ctx: CopilotContext,
@@ -1399,21 +1493,12 @@ async def _run_blocks_and_collect_debug(
},
}
- data: dict[str, Any] = {}
- for wp in all_workflow_params:
- if wp.key in user_params:
- data[wp.key] = user_params[wp.key]
- elif wp.default_value is not None:
- data[wp.key] = wp.default_value
- else:
- placeholder = _placeholder_for_parameter_type(wp.workflow_parameter_type)
- if placeholder is not None:
- data[wp.key] = placeholder
- LOG.info(
- "Auto-filled missing workflow parameter for copilot test run",
- parameter_key=wp.key,
- parameter_type=str(wp.workflow_parameter_type),
- )
+ # Multi-word/PII values the persist-time policy withholds from default_value are bound run-scoped
+ # here (WorkflowRequestBody.data, never default_value) so the test run uses the scout-proven value.
+ scout_ephemeral_values = _scout_ephemeral_values(ctx, {wp.key for wp in all_workflow_params})
+ data, ctx.unbound_required_parameter_keys = _resolve_run_data_and_unbound_keys(
+ all_workflow_params, user_params, scout_ephemeral_values
+ )
use_fresh_session = _should_use_fresh_session_for_login_first_replay(ctx, labels_to_execute, workflow)
# True when the run was threaded into a fresh session rather than the scout's debug session;
@@ -2733,7 +2818,7 @@ def _record_run_blocks_result(
def _record_adjudicated_build_test_outcome(
- copilot_ctx: Any,
+ copilot_ctx: CopilotContext,
result: dict[str, Any],
completion_verification: CompletionVerificationResult | None,
recorded_run_outcome: RecordedRunOutcome | None,
@@ -2744,18 +2829,25 @@ def _record_adjudicated_build_test_outcome(
if isinstance(result_data, dict) and isinstance(result_data.get("registered_output_parameter_values"), list)
else None
)
+ workflow_yaml = copilot_ctx.workflow_yaml
+ code_artifact_metadata = copilot_ctx.code_artifact_metadata
record_build_test_outcome(
copilot_ctx,
recorded_outcome_from_run_blocks_result(
result,
- page_evidence=getattr(copilot_ctx, "composition_page_evidence", None),
+ page_evidence=copilot_ctx.composition_page_evidence,
recorded_run_outcome=recorded_run_outcome,
completion_verification=completion_verification,
authored_structure_signature=authored_structure_signature_from_workflow(
- getattr(copilot_ctx, "workflow_yaml", None),
- getattr(copilot_ctx, "code_artifact_metadata", None),
+ workflow_yaml,
+ code_artifact_metadata,
),
registered_output_parameter_payloads=registered_output_parameter_payloads,
+ unbound_required_parameter_keys=list(copilot_ctx.unbound_required_parameter_keys),
+ block_parameter_keys=authored_block_parameter_keys_from_workflow(
+ workflow_yaml,
+ code_artifact_metadata,
+ ),
),
)
diff --git a/skyvern/forge/sdk/copilot/tools/scouting.py b/skyvern/forge/sdk/copilot/tools/scouting.py
index 937785dc7..a72ced58a 100644
--- a/skyvern/forge/sdk/copilot/tools/scouting.py
+++ b/skyvern/forge/sdk/copilot/tools/scouting.py
@@ -303,6 +303,7 @@ def _record_scouted_interaction(
source_url: str | None = None,
value: str = "",
typed_value: str = "",
+ raw_typed_value: str = "",
key: str = "",
typed_length: int = 0,
role: str = "",
@@ -325,6 +326,8 @@ def _record_scouted_interaction(
artifact["value"] = value
if typed_value:
artifact["typed_value"] = typed_value
+ if raw_typed_value:
+ artifact["raw_typed_value"] = raw_typed_value
if key:
artifact["key"] = key
if typed_length:
diff --git a/tests/unit/test_copilot_enforcement_pruning.py b/tests/unit/test_copilot_enforcement_pruning.py
index 3e7b6f068..8242f19df 100644
--- a/tests/unit/test_copilot_enforcement_pruning.py
+++ b/tests/unit/test_copilot_enforcement_pruning.py
@@ -1190,11 +1190,14 @@ def test_analyze_text_prompt_all_null_is_empty() -> None:
def _fresh_ctx_for_record() -> SimpleNamespace:
"""SimpleNamespace shaped for _record_run_blocks_result + update_repeated_failure_state.
- Uses getattr-with-default-compatible defaults so the function under test
- populates the interesting fields without tripping AttributeError on the
- downstream update_repeated_failure_state call.
+ Mirrors the AgentContext field defaults the function under test reads directly,
+ so the stub populates the interesting fields without tripping AttributeError on
+ the downstream update_repeated_failure_state call.
"""
return SimpleNamespace(
+ code_artifact_metadata={},
+ composition_page_evidence=None,
+ unbound_required_parameter_keys=[],
last_test_ok=True,
last_test_failure_reason=None,
last_test_suspicious_success=False,
diff --git a/tests/unit/test_copilot_scout_test_value_binding.py b/tests/unit/test_copilot_scout_test_value_binding.py
new file mode 100644
index 000000000..9f08e40ea
--- /dev/null
+++ b/tests/unit/test_copilot_scout_test_value_binding.py
@@ -0,0 +1,374 @@
+"""Tests for the scout->test-run value binding lane and the required_input_unbound outcome.
+
+OSS-synced: only RFC-2606 / localhost-mock placeholder data ('Jordan Avery', example.com).
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from types import SimpleNamespace
+from typing import Any
+
+from skyvern.forge.sdk.copilot.build_test_outcome import (
+ _binding_frontier_facet,
+ authored_block_parameter_keys_from_workflow,
+ recorded_outcome_from_run_blocks_result,
+)
+from skyvern.forge.sdk.copilot.code_block_synthesis import (
+ _typed_value_identity,
+ synthesize_code_block,
+)
+from skyvern.forge.sdk.copilot.reached_download_target import ReachedDownloadTarget
+from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
+from skyvern.forge.sdk.copilot.tools.run_execution import (
+ _resolve_run_data_and_unbound_keys,
+ _scout_ephemeral_values,
+)
+from skyvern.forge.sdk.copilot.tools.scouting import _record_scouted_interaction
+from skyvern.forge.sdk.workflow.models.parameter import WorkflowParameter, WorkflowParameterType
+
+SCOUT_VALUE = "Jordan Avery"
+
+
+def _wp(
+ key: str,
+ *,
+ default_value: str | None = None,
+ ptype: WorkflowParameterType = WorkflowParameterType.STRING,
+) -> WorkflowParameter:
+ now = datetime.now(timezone.utc)
+ return WorkflowParameter(
+ workflow_parameter_id=f"wp_{key}",
+ workflow_parameter_type=ptype,
+ key=key,
+ description=None,
+ workflow_id="wf_id",
+ default_value=default_value,
+ created_at=now,
+ modified_at=now,
+ )
+
+
+def _type_text(
+ accessible_name: str,
+ *,
+ raw: str,
+ typed_value: str = "",
+ index: int = 0,
+ selector: str | None = None,
+) -> dict[str, Any]:
+ interaction: dict[str, Any] = {
+ "tool_name": "type_text",
+ "source_url": "https://example.com/search",
+ "role": "textbox",
+ "accessible_name": accessible_name,
+ "typed_length": len(raw),
+ "trajectory_index": index,
+ }
+ resolved_selector = f"#{accessible_name.lower().replace(' ', '_')}" if selector is None else selector
+ if resolved_selector:
+ interaction["selector"] = resolved_selector
+ if typed_value:
+ interaction["typed_value"] = typed_value
+ if raw:
+ interaction["raw_typed_value"] = raw
+ return interaction
+
+
+class _Ctx:
+ def __init__(
+ self,
+ trajectory: list[dict[str, Any]],
+ *,
+ reached_download_target: ReachedDownloadTarget | None = None,
+ ) -> None:
+ self.scout_trajectory = trajectory
+ self.reached_download_target = reached_download_target
+
+
+def _minted_key(trajectory: list[dict[str, Any]]) -> str:
+ synthesized = synthesize_code_block(trajectory)
+ assert synthesized is not None
+ bindings = synthesized.diagnostics.typed_param_bindings
+ assert bindings
+ return bindings[0][1]
+
+
+def test_scout_binds_when_default_absent() -> None:
+ trajectory = [_type_text("Provider name", raw=SCOUT_VALUE)]
+ key = _minted_key(trajectory)
+ ephemeral = _scout_ephemeral_values(_Ctx(trajectory), {key})
+ assert ephemeral == {key: SCOUT_VALUE}
+ data, unbound = _resolve_run_data_and_unbound_keys([_wp(key)], {}, ephemeral)
+ assert data[key] == SCOUT_VALUE
+ assert unbound == []
+
+
+def test_scout_binds_when_default_empty_string() -> None:
+ trajectory = [_type_text("Provider name", raw=SCOUT_VALUE)]
+ key = _minted_key(trajectory)
+ ephemeral = _scout_ephemeral_values(_Ctx(trajectory), {key})
+ data, unbound = _resolve_run_data_and_unbound_keys([_wp(key, default_value="")], {}, ephemeral)
+ assert data[key] == SCOUT_VALUE
+ assert unbound == []
+
+
+def test_non_empty_default_wins_over_scout() -> None:
+ trajectory = [_type_text("Provider name", raw=SCOUT_VALUE)]
+ key = _minted_key(trajectory)
+ ephemeral = _scout_ephemeral_values(_Ctx(trajectory), {key})
+ data, unbound = _resolve_run_data_and_unbound_keys([_wp(key, default_value="cardiology")], {}, ephemeral)
+ assert data[key] == "cardiology"
+ assert unbound == []
+
+
+def test_user_param_wins_over_scout() -> None:
+ trajectory = [_type_text("Provider name", raw=SCOUT_VALUE)]
+ key = _minted_key(trajectory)
+ ephemeral = _scout_ephemeral_values(_Ctx(trajectory), {key})
+ data, unbound = _resolve_run_data_and_unbound_keys([_wp(key)], {key: "user-supplied"}, ephemeral)
+ assert data[key] == "user-supplied"
+ assert unbound == []
+
+
+def test_same_name_selectorless_sibling_drift_falls_safe() -> None:
+ trajectory = [
+ _type_text("Provider name", raw="Casey Nguyen", selector="", index=0),
+ _type_text("Provider name", raw=SCOUT_VALUE, selector="#provider", index=1),
+ ]
+ strict = synthesize_code_block(trajectory, strict_selectors=True)
+ lenient = synthesize_code_block(trajectory, strict_selectors=False)
+ assert strict is not None and lenient is not None
+ strict_key = dict(strict.diagnostics.typed_param_bindings)[1]
+ assert dict(lenient.diagnostics.typed_param_bindings)[1] != strict_key
+ ephemeral = _scout_ephemeral_values(_Ctx(trajectory), {strict_key})
+ assert ephemeral == {}
+
+
+def test_distinct_named_field_binds_despite_dropped_selectorless_sibling() -> None:
+ trajectory = [
+ _type_text("First name", raw="Casey Nguyen", selector="", index=0),
+ _type_text("Last name", raw=SCOUT_VALUE, selector="#last", index=1),
+ ]
+ ephemeral = _scout_ephemeral_values(_Ctx(trajectory), {"last_name"})
+ assert ephemeral == {"last_name": SCOUT_VALUE}
+
+
+def test_no_scout_value_falls_to_placeholder_and_records_unbound() -> None:
+ data, unbound = _resolve_run_data_and_unbound_keys([_wp("search")], {}, {})
+ assert data["search"] == ""
+ assert unbound == ["search"]
+
+
+def test_empty_string_default_no_scout_records_unbound() -> None:
+ data, unbound = _resolve_run_data_and_unbound_keys([_wp("search", default_value="")], {}, {})
+ assert unbound == ["search"]
+
+
+def test_secret_like_scout_value_not_bound() -> None:
+ trajectory = [_type_text("Login", raw="[email protected]:hunter2Pass")]
+ key = _minted_key(trajectory)
+ ephemeral = _scout_ephemeral_values(_Ctx(trajectory), {key})
+ assert ephemeral == {}
+
+
+def test_secret_field_context_scout_value_not_bound() -> None:
+ trajectory = [_type_text("Password", raw="Redwood", selector="#password")]
+ key = _minted_key(trajectory)
+ ephemeral = _scout_ephemeral_values(_Ctx(trajectory), {key})
+ assert ephemeral == {}
+
+
+def test_ambiguous_duplicate_value_falls_safe_to_unbound() -> None:
+ trajectory = [
+ _type_text("First field", raw=SCOUT_VALUE, index=0),
+ _type_text("Second field", raw=SCOUT_VALUE, index=1),
+ ]
+ synthesized = synthesize_code_block(trajectory)
+ assert synthesized is not None
+ keys = {key for _, key in synthesized.diagnostics.typed_param_bindings}
+ ephemeral = _scout_ephemeral_values(_Ctx(trajectory), keys)
+ assert ephemeral == {}
+
+
+def test_key_not_in_workflow_falls_safe() -> None:
+ trajectory = [_type_text("Provider name", raw=SCOUT_VALUE)]
+ ephemeral = _scout_ephemeral_values(_Ctx(trajectory), {"unrelated_key"})
+ assert ephemeral == {}
+
+
+def test_empty_trajectory_no_crash_no_bind() -> None:
+ assert _scout_ephemeral_values(_Ctx([]), {"search"}) == {}
+
+
+def test_record_scouted_interaction_carries_raw_value_into_trajectory() -> None:
+ ctx = SimpleNamespace(
+ scouted_interactions=[],
+ scout_trajectory=[],
+ last_evaluate_actionable_signature=None,
+ last_evaluate_actionable_url=None,
+ latest_evaluate_result_composition_steer=None,
+ )
+ _record_scouted_interaction(
+ ctx,
+ tool_name="type_text",
+ selector="#provider",
+ source_url="https://example.com/search",
+ typed_length=len(SCOUT_VALUE),
+ typed_value="",
+ raw_typed_value=SCOUT_VALUE,
+ role="textbox",
+ accessible_name="Provider name",
+ )
+ assert ctx.scout_trajectory[0]["raw_typed_value"] == SCOUT_VALUE
+ assert "typed_value" not in ctx.scout_trajectory[0]
+
+
+def test_raw_value_excluded_from_typed_value_identity() -> None:
+ interaction = _type_text("Provider name", raw=SCOUT_VALUE)
+ assert _typed_value_identity(interaction) is None
+
+
+def test_raw_value_not_promoted_to_default_value() -> None:
+ trajectory = [_type_text("Provider name", raw=SCOUT_VALUE)]
+ synthesized = synthesize_code_block(trajectory)
+ assert synthesized is not None
+ assert all("default_value" not in parameter for parameter in synthesized.parameters)
+
+
+_WORKFLOW_YAML = """
+workflow_definition:
+ parameters:
+ - key: search_by_specialty
+ parameter_type: workflow
+ blocks:
+ - block_type: code
+ label: search_block
+ code: "await page.fill('#q', str(search_by_specialty))"
+ parameter_keys:
+ - search_by_specialty
+""".strip()
+
+
+def _failed_run_result(*, failure_reason: str, label: str = "search_block", ok: bool = False) -> dict[str, Any]:
+ return {
+ "ok": ok,
+ "data": {
+ "workflow_run_id": "wr_test",
+ "overall_status": "failed",
+ "blocks": [{"label": label, "status": "failed", "failure_reason": failure_reason}],
+ },
+ }
+
+
+_LOCATOR_WAIT_REASON = "Timeout 30000ms exceeded waiting for locator('#coastalCard') to be visible"
+
+
+def test_authored_block_parameter_keys_from_workflow() -> None:
+ mapping = authored_block_parameter_keys_from_workflow(_WORKFLOW_YAML)
+ assert mapping == {"search_block": ["search_by_specialty"]}
+
+
+def test_required_input_unbound_fires_when_failed_block_references_key() -> None:
+ outcome = recorded_outcome_from_run_blocks_result(
+ _failed_run_result(failure_reason=_LOCATOR_WAIT_REASON),
+ unbound_required_parameter_keys=["search_by_specialty"],
+ block_parameter_keys={"search_block": ["search_by_specialty"]},
+ )
+ assert outcome is not None
+ assert outcome.reason_code == "required_input_unbound"
+ assert _binding_frontier_facet(outcome) == "amend_in_place"
+
+
+def test_required_input_unbound_authoritative_on_non_locator_wait_failure() -> None:
+ outcome = recorded_outcome_from_run_blocks_result(
+ _failed_run_result(failure_reason="some unrelated runtime error"),
+ unbound_required_parameter_keys=["search_by_specialty"],
+ block_parameter_keys={"search_block": ["search_by_specialty"]},
+ )
+ assert outcome is not None
+ assert outcome.reason_code == "required_input_unbound"
+ assert outcome.is_authoritative
+ assert outcome.structural_key is not None
+
+
+def test_required_input_unbound_identity_is_key_order_insensitive() -> None:
+ block_parameter_keys = {"search_block": ["search_by_specialty", "search_by_location"]}
+ ascending = recorded_outcome_from_run_blocks_result(
+ _failed_run_result(failure_reason="some unrelated runtime error"),
+ unbound_required_parameter_keys=["search_by_specialty", "search_by_location"],
+ block_parameter_keys=block_parameter_keys,
+ )
+ descending = recorded_outcome_from_run_blocks_result(
+ _failed_run_result(failure_reason="some unrelated runtime error"),
+ unbound_required_parameter_keys=["search_by_location", "search_by_specialty"],
+ block_parameter_keys=block_parameter_keys,
+ )
+ assert ascending is not None and descending is not None
+ assert ascending.structural_key == descending.structural_key
+
+
+def test_terminal_challenge_blocker_wins_over_required_input_unbound() -> None:
+ outcome = recorded_outcome_from_run_blocks_result(
+ _failed_run_result(failure_reason=_LOCATOR_WAIT_REASON),
+ recorded_run_outcome=RecordedRunOutcome(verdict="not_demonstrated", reason_code="terminal_challenge_blocker"),
+ unbound_required_parameter_keys=["search_by_specialty"],
+ block_parameter_keys={"search_block": ["search_by_specialty"]},
+ )
+ assert outcome is not None
+ assert outcome.reason_code == "terminal_challenge_blocker"
+
+
+def test_demonstrated_run_wins_over_required_input_unbound() -> None:
+ outcome = recorded_outcome_from_run_blocks_result(
+ _failed_run_result(failure_reason=_LOCATOR_WAIT_REASON),
+ recorded_run_outcome=RecordedRunOutcome(verdict="demonstrated"),
+ unbound_required_parameter_keys=["search_by_specialty"],
+ block_parameter_keys={"search_block": ["search_by_specialty"]},
+ )
+ assert outcome is not None
+ assert outcome.reason_code == "verified_success"
+
+
+def test_not_evaluated_run_wins_over_required_input_unbound() -> None:
+ outcome = recorded_outcome_from_run_blocks_result(
+ _failed_run_result(failure_reason=_LOCATOR_WAIT_REASON),
+ recorded_run_outcome=RecordedRunOutcome(verdict="not_evaluated"),
+ unbound_required_parameter_keys=["search_by_specialty"],
+ block_parameter_keys={"search_block": ["search_by_specialty"]},
+ )
+ assert outcome is not None
+ assert outcome.reason_code != "required_input_unbound"
+
+
+def test_required_input_unbound_not_fired_when_block_does_not_reference_key() -> None:
+ outcome = recorded_outcome_from_run_blocks_result(
+ _failed_run_result(failure_reason=_LOCATOR_WAIT_REASON),
+ unbound_required_parameter_keys=["some_other_key"],
+ block_parameter_keys={"search_block": ["search_by_specialty"]},
+ )
+ assert outcome is not None
+ assert outcome.reason_code == "runtime_block_failure"
+ assert _binding_frontier_facet(outcome) == "selector_frontier"
+
+
+def test_required_input_unbound_not_fired_when_payload_missing() -> None:
+ outcome = recorded_outcome_from_run_blocks_result(
+ _failed_run_result(failure_reason=_LOCATOR_WAIT_REASON),
+ unbound_required_parameter_keys=["search_by_specialty"],
+ block_parameter_keys={},
+ )
+ assert outcome is not None
+ assert outcome.reason_code == "runtime_block_failure"
+
+
+def test_required_input_unbound_not_fired_on_success() -> None:
+ result = _failed_run_result(failure_reason=_LOCATOR_WAIT_REASON, ok=True)
+ result["data"]["overall_status"] = "completed"
+ outcome = recorded_outcome_from_run_blocks_result(
+ result,
+ unbound_required_parameter_keys=["search_by_specialty"],
+ block_parameter_keys={"search_block": ["search_by_specialty"]},
+ )
+ if outcome is not None:
+ assert outcome.reason_code != "required_input_unbound"