Skip to content

Automate browser based workflows with AI

fix(SKY-12164): repair incomplete workflow outputs on first validation (#7308)

Andrew Neilson417cac70da25
diff --git a/skyvern/forge/sdk/copilot/mcp_adapter.py b/skyvern/forge/sdk/copilot/mcp_adapter.py
index e50dcd688..26875ceda 100644
--- a/skyvern/forge/sdk/copilot/mcp_adapter.py
+++ b/skyvern/forge/sdk/copilot/mcp_adapter.py
@@ -72,6 +72,7 @@ _POST_HOOK_CONTEXT_ROLLBACK_FIELDS = (
"last_evaluate_actionable_signature",
"last_evaluate_actionable_url",
"latest_evaluate_result_composition_steer",
+ "latest_evaluate_result_composition_signature",
"last_auto_acted_signature",
"reached_download_target",
"synthesized_block_offered",
diff --git a/skyvern/forge/sdk/copilot/output_contracts.py b/skyvern/forge/sdk/copilot/output_contracts.py
index 2dc3e67ab..11d45ab4c 100644
--- a/skyvern/forge/sdk/copilot/output_contracts.py
+++ b/skyvern/forge/sdk/copilot/output_contracts.py
@@ -62,6 +62,7 @@ class OutputContractActuationEvidence:
consumed_run_output_observed: bool = False
consumed_run_bound_required_path: bool = False
consumed_run_carried_page_extraction: bool = False
+ loaded_result_source_producible: bool = False
@dataclass(frozen=True)
@@ -95,12 +96,15 @@ def resolve_output_contract_actuation(
grantable_source = (
evidence.advisory_run_grantable and evidence.advisory_state != OutputContractAdvisoryState.CONSUMED
)
- observable_source = evidence.observed_required_values or not evidence.click_only_spine
+ observable_source = (
+ evidence.observed_required_values or evidence.loaded_result_source_producible or not evidence.click_only_spine
+ )
producible_source = observable_source or grantable_source
progressed = evidence.actuation_progress_exhausted or evidence.prior_actuation
if (
evidence.click_only_spine
and not evidence.observed_required_values
+ and not evidence.loaded_result_source_producible
and evidence.declick_attempt_failed
and not grantable_source
):
diff --git a/skyvern/forge/sdk/copilot/result_evidence.py b/skyvern/forge/sdk/copilot/result_evidence.py
index 2c421e770..6ea4a4bad 100644
--- a/skyvern/forge/sdk/copilot/result_evidence.py
+++ b/skyvern/forge/sdk/copilot/result_evidence.py
@@ -85,6 +85,8 @@ class LoadedResultCompositionEvidence:
table_result_container_count: int
targets: tuple[LoadedResultCompositionTarget, ...] = ()
structure_signature: str = ""
+ source_tool: str = ""
+ source_url: str = ""
def _is_meaningful_data(value: Any, *, _depth: int = 0) -> bool:
@@ -326,6 +328,9 @@ def _cap_loaded_result_composition_summary(summary: dict[str, object]) -> dict[s
def loaded_result_composition_evidence_from_page(
evidence: Mapping[str, Any],
+ *,
+ source_tool: str = "",
+ source_url: str = "",
) -> LoadedResultCompositionEvidence | None:
raw_containers = evidence.get("result_containers")
if not isinstance(raw_containers, list):
@@ -342,9 +347,40 @@ def loaded_result_composition_evidence_from_page(
table_result_container_count=sum(1 for container in meaningful if _result_container_is_table(container)),
targets=targets,
structure_signature=_structure_signature([_target_signature_payload(target) for target in targets]),
+ source_tool=_bounded_text(source_tool, _MAX_SELECTOR_CHARS) if source_tool.strip() else "",
+ source_url=_bounded_text(source_url, _MAX_SELECTOR_CHARS) if source_url.strip() else "",
)
+def loaded_result_source_producible(
+ evidence: LoadedResultCompositionEvidence | None,
+ *,
+ target_code: str | None = None,
+) -> bool:
+ if evidence is None or evidence.source_tool != "evaluate" or not evidence.source_url.strip():
+ return False
+ if evidence.result_container_count < 1 or not evidence.targets or not evidence.structure_signature:
+ return False
+ if evidence.result_container_count < len(evidence.targets):
+ return False
+ for target in evidence.targets:
+ if target.structure_signature != _structure_signature(_target_signature_payload(target)):
+ return False
+ expected_signature = _structure_signature([_target_signature_payload(target) for target in evidence.targets])
+ if evidence.structure_signature != expected_signature:
+ return False
+ if target_code is not None:
+ observed_selectors = {
+ selector
+ for target in evidence.targets
+ for selector in (target.selector, target.row_selector)
+ if selector.strip()
+ }
+ if not observed_selectors or not any(selector in target_code for selector in observed_selectors):
+ return False
+ return True
+
+
def loaded_result_composition_target_summary(evidence: LoadedResultCompositionEvidence) -> dict[str, object]:
summary = _summary_with_targets(evidence, [_target_summary(target) for target in evidence.targets])
return _cap_loaded_result_composition_summary(summary)
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index 25eaae837..c90dd4453 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -360,6 +360,7 @@ class AgentContext:
last_evaluate_actionable_signature: str | None = None
last_evaluate_actionable_url: str | None = None
latest_evaluate_result_composition_steer: LoadedResultCompositionEvidence | None = None
+ latest_evaluate_result_composition_signature: str | None = None
last_auto_acted_signature: str | None = None
observed_browser_urls: list[str] = field(default_factory=list)
# Ephemeral within-turn scout captures; not persisted across turns.
diff --git a/skyvern/forge/sdk/copilot/tools/__init__.py b/skyvern/forge/sdk/copilot/tools/__init__.py
index 7cae2265f..ab8fd8205 100644
--- a/skyvern/forge/sdk/copilot/tools/__init__.py
+++ b/skyvern/forge/sdk/copilot/tools/__init__.py
@@ -706,6 +706,7 @@ async def update_and_run_blocks_tool(
copilot_ctx,
allow_missing_credentials=skip_run_after_update,
allow_static_output_uncertainty=True,
+ formation_prepared=True,
)
_record_workflow_update_result(copilot_ctx, update_result, prior_definition)
diff --git a/skyvern/forge/sdk/copilot/tools/scouting.py b/skyvern/forge/sdk/copilot/tools/scouting.py
index d8c55bbb8..4e3321445 100644
--- a/skyvern/forge/sdk/copilot/tools/scouting.py
+++ b/skyvern/forge/sdk/copilot/tools/scouting.py
@@ -44,6 +44,7 @@ from skyvern.forge.sdk.copilot.reached_download_target import (
)
from skyvern.forge.sdk.copilot.reached_download_target import guidance_for as _reached_download_guidance_for
from skyvern.forge.sdk.copilot.result_evidence import (
+ LoadedResultCompositionEvidence,
loaded_result_composition_evidence_from_page,
loaded_result_composition_target_summary,
)
@@ -596,6 +597,25 @@ def _scout_act_observe_no_payload_result(*, started: float, timeout_seconds: flo
return "timeout" if time.monotonic() - started >= timeout_seconds else "no_payload"
+def _mint_current_loaded_result_source(
+ ctx: AgentContext,
+ page_evidence: dict[str, Any] | None,
+ *,
+ url: str,
+) -> LoadedResultCompositionEvidence | None:
+ if page_evidence is None:
+ return None
+ loaded_results = loaded_result_composition_evidence_from_page(
+ page_evidence,
+ source_tool="evaluate",
+ source_url=url,
+ )
+ if loaded_results is not None:
+ ctx.latest_evaluate_result_composition_steer = loaded_results
+ ctx.latest_evaluate_result_composition_signature = None
+ return loaded_results
+
+
async def _scout_act_observe_page_evidence(ctx: AgentContext, *, url: str) -> dict[str, Any] | None:
"""Run the bounded page-side extractor right after a scout interaction.
@@ -861,6 +881,7 @@ def _reset_evaluate_tracker(ctx: AgentContext) -> None:
ctx.last_evaluate_actionable_signature = None
ctx.last_evaluate_actionable_url = None
ctx.latest_evaluate_result_composition_steer = None
+ ctx.latest_evaluate_result_composition_signature = None
def _actionable_target_identities(evidence: dict[str, Any]) -> list[tuple[str, str]]:
@@ -1212,7 +1233,7 @@ async def _maybe_steer_evaluate_to_action(
_reset_evaluate_tracker(ctx)
return False
record_scouted_output_coverage(ctx, parsed)
- loaded_results = loaded_result_composition_evidence_from_page(parsed)
+ loaded_results = _mint_current_loaded_result_source(ctx, parsed, url=url)
if loaded_results is not None:
_reset_evaluate_tracker(ctx)
ctx.latest_evaluate_result_composition_steer = loaded_results
@@ -1232,6 +1253,7 @@ async def _maybe_steer_evaluate_to_action(
# The result is patched in-place; returning False keeps the normal tool-loop guard active.
return False
ctx.latest_evaluate_result_composition_steer = None
+ ctx.latest_evaluate_result_composition_signature = None
identities = _actionable_target_identities(parsed)
if not identities:
_reset_evaluate_tracker(ctx)
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index 87a458ee2..08decf662 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -121,6 +121,7 @@ from skyvern.forge.sdk.copilot.request_policy import (
_coerce_requested_output_evidence_source,
_is_judgment_boolean_criterion,
)
+from skyvern.forge.sdk.copilot.result_evidence import loaded_result_source_producible
from skyvern.forge.sdk.copilot.runtime import (
AgentContext,
AuthorTimeGateAblationPayload,
@@ -1830,7 +1831,6 @@ _SEPARATED_SPINE_SHAPE_REQUIRED_REASON_CODE = "separated_spine_shape_required"
_SEPARATED_BROWSER_SPINE_PLUS_EXTRACTION_STRUCTURE = "separated_browser_spine_plus_extraction"
_OUTPUT_CONTRACT_REJECT_REASON_CODE = "output_contract_required"
_OUTPUT_CONTRACT_REJECT_BUDGET_REASON_CODE = "output_contract_reject_budget_exhausted"
-_MAX_OUTPUT_CONTRACT_STEERING_REJECTS = 2
_MAX_OUTPUT_CONTRACT_REJECTS = 4
_MAX_OUTPUT_CONTRACT_DEFERRALS = 3
_MAX_OUTPUT_CONTRACT_ACTUATIONS_WITHOUT_RUN = 3
@@ -2433,34 +2433,6 @@ def _declaration_envelope_paths(declaration_paths: set[str]) -> set[str]:
return declaration_paths | {_output_path_root(path) for path in declaration_paths}
-def _code_return_is_static_advisory_candidate(
- code: str,
- required_paths: set[str],
- declaration_paths: set[str] | None = None,
-) -> bool:
- declaration_paths = declaration_paths or set()
- if not (required_paths - declaration_paths):
- return False
- declaration_envelope = _declaration_envelope_paths(declaration_paths)
- produced_paths = _code_block_produced_output_paths(code)
- if produced_paths - declaration_envelope:
- return False
- produced = _code_block_produced_output_roots(code)
- # A stamped declaration-only return (e.g. blocker: None) keeps the block abstained for the
- # observation lane; only observation-lane production disqualifies the advisory.
- declaration_only_production = bool(produced_paths) and produced_paths <= declaration_envelope
- if not produced.abstained and not declaration_only_production:
- return False
- try:
- tree = ast.parse(textwrap.dedent(code).strip() or "pass")
- except SyntaxError:
- return False
- for node in _iter_top_level_scope(tree.body):
- if isinstance(node, ast.Return) and node.value is not None and _expr_is_flat_string(node.value, set()):
- return False
- return True
-
-
def _evaluate_output_contract_for_code_block(
ctx: AgentContext,
workflow_yaml: str,
@@ -2537,17 +2509,14 @@ def _evaluate_output_contract_for_code_block(
for violation in shape_violations
if not (violation == _SEPARATED_SPINE_SHAPE_REQUIRED_REASON_CODE and separated_spine_advisory_run)
]
- post_steering_static_return_advisory = (
+ actuated_static_return_advisory = (
allow_static_return_advisory
and bool(missing_observation_return_paths)
and not missing_metadata_paths
and not missing_schema_paths
and not run_gating_shape_violations
and target_block is not None
- and (
- _output_contract_advisory_granted(ctx, signature)
- or _output_contract_reject_count(ctx, signature) >= _MAX_OUTPUT_CONTRACT_STEERING_REJECTS
- )
+ and _output_contract_advisory_granted(ctx, signature)
)
static_return_advisory = (
allow_static_return_advisory
@@ -2556,12 +2525,7 @@ def _evaluate_output_contract_for_code_block(
and not missing_schema_paths
and not run_gating_shape_violations
and target_block is not None
- and (
- post_steering_static_return_advisory
- or _code_return_is_static_advisory_candidate(
- str(target_block.get("code") or ""), required_paths, declaration_paths
- )
- )
+ and actuated_static_return_advisory
)
separated_spine_run_eligible = (
separated_spine_advisory_run
@@ -2615,7 +2579,7 @@ def _evaluate_output_contract_for_code_block(
"missing_extraction_schema_paths": missing_schema_paths,
"missing_code_return_paths": effective_missing_return_paths,
"static_return_advisory_paths": missing_observation_return_paths if static_return_advisory else [],
- "post_steering_static_return_advisory": post_steering_static_return_advisory,
+ "actuated_static_return_advisory": actuated_static_return_advisory,
"shape_violations": shape_violations,
"can_attempt_run": run_eligible,
"reject_reason": "" if run_eligible else _OUTPUT_CONTRACT_REJECT_REASON_CODE,
@@ -2702,22 +2666,22 @@ def _adjudicate_output_contract_ladder_after_reject(
*,
workflow_yaml: str,
current_fingerprint: str,
-) -> None:
- """Run the actuation ladder at the shared reject-counting seam so a signature whose imposition
- early-outed against a resolvable owner block (the steering-reject wait) still advances toward an
- advisory-consumed run or a typed terminal within the existing caps, keeping the loop/churn defer
- bounded. A bail with no owner block is left to the owner-directive path, not granted an inert run."""
+) -> OutputContractActuation | None:
+ """Run the actuation ladder at the shared deficiency seam so a signature whose formation
+ remains incomplete advances toward an advisory-consumed run or a typed terminal within the
+ existing caps, keeping the loop/churn defer bounded. A bail with no owner block is left to the
+ owner-directive path, not granted an inert run."""
if ctx.turn_halt is not None or ctx.output_contract_bail_actuated_this_call:
- return
+ return None
signature = evaluation.canonical_signature
block_label = evaluation.block_label
if not signature or not block_label:
- return
+ return None
if _output_contract_advisory_state(ctx, signature) in {
OutputContractAdvisoryState.GRANTED,
OutputContractAdvisoryState.CONSUMED,
}:
- return
+ return None
block = _workflow_yaml_code_blocks_by_label(workflow_yaml).get(block_label)
target_code = str(block.get("code") or "") if block is not None else ""
blockers = list(evaluation.shape_violations) or [_OUTPUT_CONTRACT_REJECT_REASON_CODE]
@@ -2740,6 +2704,9 @@ def _adjudicate_output_contract_ladder_after_reject(
signature=signature,
blockers=blockers,
)
+ elif actuation.kind == OutputContractActuationKind.STRUCTURE_DIRECTIVE:
+ _record_armed_directive_fingerprint(ctx, signature, current_fingerprint)
+ return actuation
def _record_output_contract_reject(
@@ -2759,6 +2726,24 @@ def _record_output_contract_reject(
payload = dict(evaluation.payload)
payload["output_contract_reject_count"] = count
payload["output_contract_reject_budget"] = _MAX_OUTPUT_CONTRACT_REJECTS
+ actuation = _adjudicate_output_contract_ladder_after_reject(
+ ctx,
+ evaluation,
+ workflow_yaml=workflow_yaml,
+ current_fingerprint=authored_structural_fingerprint,
+ )
+ if actuation is not None:
+ payload["output_contract_actuation"] = actuation.kind.value
+ if actuation.kind != OutputContractActuationKind.STRUCTURE_DIRECTIVE:
+ latest_outcome = ctx.latest_recorded_build_test_outcome
+ if (
+ isinstance(latest_outcome, RecordedBuildTestOutcome)
+ and latest_outcome.phase == "author_time_reject"
+ and latest_outcome.reason_code == "metadata_reject"
+ ):
+ # The typed adjudication now owns this same deficiency. Keep the historical reject for
+ # ceiling/liveness evidence, but do not render it again as a fresh rejection next turn.
+ record_build_test_outcome(ctx, None)
if count >= _MAX_OUTPUT_CONTRACT_REJECTS:
if run_backed_repair_evidence_exists(ctx):
payload["reason_code"] = _OUTPUT_CONTRACT_REJECT_BUDGET_REASON_CODE
@@ -2783,28 +2768,23 @@ def _record_output_contract_reject(
output_contract_reject_count=count,
output_contract_deferral_count=deferral_count,
)
- structural_payload = _output_contract_author_time_structural_payload(
- ctx,
- evaluation.required_paths,
- block_label=evaluation.block_label,
- )
- _record_author_time_reject_outcome(
- ctx,
- reason_code="metadata_reject",
- summary=summary,
- structural_payload=structural_payload or payload,
- block_labels=[evaluation.block_label] if evaluation.block_label else [],
- missing_requested_output_facts=payload.get("missing_requested_output_facts")
- if isinstance(payload.get("missing_requested_output_facts"), list)
- else None,
- )
- _adjudicate_output_contract_ladder_after_reject(
- ctx,
- evaluation,
- workflow_yaml=workflow_yaml,
- current_fingerprint=authored_structural_fingerprint,
- )
- _record_code_authoring_guardrail_reject(ctx)
+ if actuation is None or actuation.kind == OutputContractActuationKind.STRUCTURE_DIRECTIVE:
+ structural_payload = _output_contract_author_time_structural_payload(
+ ctx,
+ evaluation.required_paths,
+ block_label=evaluation.block_label,
+ )
+ _record_author_time_reject_outcome(
+ ctx,
+ reason_code="metadata_reject",
+ summary=summary,
+ structural_payload=structural_payload or payload,
+ block_labels=[evaluation.block_label] if evaluation.block_label else [],
+ missing_requested_output_facts=payload.get("missing_requested_output_facts")
+ if isinstance(payload.get("missing_requested_output_facts"), list)
+ else None,
+ )
+ _record_code_authoring_guardrail_reject(ctx)
return payload
@@ -3056,8 +3036,7 @@ def _scaffold_metadata_contract_for_update(
source=contract.source,
reason_code=contract.reason_code,
declaration_paths=contract.declaration_paths,
- allow_missing_static_return=_output_contract_advisory_granted(ctx, signature)
- or _output_contract_reject_count(ctx, signature) >= _MAX_OUTPUT_CONTRACT_STEERING_REJECTS,
+ allow_missing_static_return=_output_contract_advisory_granted(ctx, signature),
)
return scaffolded, scaffolded is not raw_code_artifact_metadata
@@ -3229,7 +3208,6 @@ def _attempt_separated_spine_split(
return _SpineSplitOutcome(None, ["target_block_not_resolved_in_parsed"], None)
reconciliation = _reconcile_synthesized_parameters(
- ctx=ctx,
parsed=parsed,
code_block=code_block,
submitted_code=target_code,
@@ -3402,9 +3380,13 @@ def _observed_required_output_values(ctx: AgentContext, required_paths: set[str]
def _prior_output_contract_actuation(ctx: AgentContext, signature: str) -> bool:
- if _output_contract_reject_count(ctx, signature) >= 1:
- return True
- return bool(ctx.output_contract_imposed_since_last_reject_by_signature.get(signature))
+ return bool(
+ ctx.output_contract_imposed_since_last_reject_by_signature.get(signature)
+ or ctx.output_contract_armed_directive_fingerprint_by_signature.get(signature)
+ or signature in ctx.output_contract_output_owner_directive_candidates_by_signature
+ or ctx.output_contract_actuation_count_by_signature.get(signature)
+ or _output_contract_advisory_state(ctx, signature) != OutputContractAdvisoryState.UNUSED
+ )
def _prior_output_contract_directive_unconsumed(ctx: AgentContext, signature: str, current_fingerprint: str) -> bool:
@@ -3699,7 +3681,14 @@ def _actuate_output_contract_bail(
) -> OutputContractActuation:
click_only_spine = _output_contract_click_only_spine(target_code, declaration_paths)
observed_required_values = _observed_required_output_values(ctx, required_paths)
- if not click_only_spine or observed_required_values:
+ loaded_result_evidence = ctx.latest_evaluate_result_composition_steer
+ result_source_producible = loaded_result_source_producible(loaded_result_evidence, target_code=target_code)
+ claimed_signature = ctx.latest_evaluate_result_composition_signature
+ if result_source_producible and claimed_signature not in (None, signature):
+ result_source_producible = False
+ elif result_source_producible:
+ ctx.latest_evaluate_result_composition_signature = signature
+ if not click_only_spine or observed_required_values or result_source_producible:
_clear_declick_attempt(ctx, signature)
evidence = OutputContractActuationEvidence(
imposed_available=False,
@@ -3714,6 +3703,7 @@ def _actuate_output_contract_bail(
consumed_run_output_observed=bool(ctx.output_contract_run_output_observed_by_signature.get(signature)),
consumed_run_bound_required_path=bool(ctx.output_contract_run_bound_required_path_by_signature.get(signature)),
consumed_run_carried_page_extraction=_page_extraction_imposed(ctx, signature),
+ loaded_result_source_producible=result_source_producible,
)
actuation = resolve_output_contract_actuation(
family=classify_output_contract_bail_family(blockers),
@@ -3728,6 +3718,16 @@ def _actuate_output_contract_bail(
"canonical_required_child_paths": sorted(required_paths),
"advisory_state": evidence.advisory_state.value,
"advisory_run_grantable": evidence.advisory_run_grantable,
+ "loaded_result_source_producible": evidence.loaded_result_source_producible,
+ "loaded_result_structure_signature": loaded_result_evidence.structure_signature
+ if result_source_producible and loaded_result_evidence is not None
+ else "",
+ "loaded_result_source_tool": loaded_result_evidence.source_tool
+ if result_source_producible and loaded_result_evidence is not None
+ else "",
+ "loaded_result_source_url": loaded_result_evidence.source_url
+ if result_source_producible and loaded_result_evidence is not None
+ else "",
}
ctx.output_contract_bail_actuated_this_call = True
if record_author_time_gate_ablation_event(
@@ -3900,7 +3900,7 @@ def _impose_output_contract_envelope_after_steering(
reason_code=reason_code,
required_paths=required_paths,
)
- # The stamp precedes the steering-reject wait and ignores advisory state so no
+ # The stamp precedes actuation and ignores advisory state so no
# acceptance route can persist a block that omits the declaration paths.
workflow_yaml, declaration_stamped = _stamp_declaration_contract_defaults(
ctx,
@@ -3911,10 +3911,6 @@ def _impose_output_contract_envelope_after_steering(
)
_reopen_dispatch_lacked_bound_extraction(ctx, signature)
runtime_attempt_key = _runtime_output_repair_attempt_key(ctx, workflow_yaml, required_paths, source, reason_code)
- if _output_contract_reject_count(
- ctx, signature
- ) < _MAX_OUTPUT_CONTRACT_STEERING_REJECTS and _runtime_output_repair_attempt_recorded(ctx, runtime_attempt_key):
- return workflow_yaml, raw_code_artifact_metadata, declaration_stamped
label, owner_labels = _target_output_contract_block_label(
ctx,
workflow_yaml,
@@ -4025,10 +4021,9 @@ def _impose_output_contract_envelope_after_steering(
if scaffolded:
_record_runtime_output_repair_attempt(ctx, runtime_attempt_key)
LOG.info(
- "copilot_output_contract_envelope_imposed_after_steering",
+ "copilot_output_contract_formation_applied",
block_label=label,
canonical_output_contract_signature=signature,
- steering_reject_count=_output_contract_reject_count(ctx, signature),
return_envelope_applied=False,
metadata_scaffold_applied=True,
)
@@ -4095,16 +4090,6 @@ def _metadata_contract_run_preflight_reject(
workflow_yaml: str,
raw_code_artifact_metadata: object,
) -> dict[str, Any] | None:
- workflow_yaml, raw_code_artifact_metadata, _ = _impose_output_contract_envelope_after_steering(
- ctx,
- workflow_yaml,
- raw_code_artifact_metadata,
- )
- raw_code_artifact_metadata, _ = _scaffold_metadata_contract_for_update(
- ctx,
- workflow_yaml,
- raw_code_artifact_metadata,
- )
convergence_reject = _recorded_outcome_convergence_reject(
ctx,
workflow_yaml=workflow_yaml,
@@ -4173,7 +4158,7 @@ def _metadata_contract_run_preflight_reject(
authored_structural_fingerprint=authored_fingerprint,
workflow_yaml=workflow_yaml,
)
- if advisory_granted:
+ if advisory_granted or _output_contract_advisory_granted(ctx, evaluation.canonical_signature):
_arm_pending_run_evidence(ctx, evaluation.canonical_signature, set(evaluation.observation_paths))
return None
if evaluation.repair_context is not None:
@@ -7067,37 +7052,6 @@ def _selector_join_parameter_alias_by_authored_fill(
return next(iter(matches)) if len(matches) == 1 else None
-def _synthesized_parameter_binding_reject_count(ctx: AgentContext | None) -> int:
- if ctx is None:
- return 0
- required_paths = _output_contract_required_paths_source(ctx).union
- if required_paths:
- contract_count = _output_contract_reject_count(
- ctx,
- _stable_output_contract_key(_output_contract_scope_key(ctx), required_paths),
- )
- if contract_count:
- return contract_count
- history = getattr(ctx, "recorded_build_test_outcome_history", None)
- if isinstance(history, list):
- count = 0
- for item in history:
- if (
- isinstance(item, Mapping)
- and item.get("reason_code") == _SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_REASON_CODE
- ):
- count += 1
- if count:
- return count
- latest = getattr(ctx, "latest_recorded_build_test_outcome", None)
- if (
- isinstance(latest, RecordedBuildTestOutcome)
- and latest.reason_code == _SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_REASON_CODE
- ):
- return 1
- return 0
-
-
def _drop_parameter_key(parameters: list[Any], key: str) -> None:
parameters[:] = [
parameter
@@ -7106,11 +7060,6 @@ def _drop_parameter_key(parameters: list[Any], key: str) -> None:
]
-def _impose_required_string_parameter(parameters: list[Any], key: str) -> None:
- _drop_parameter_key(parameters, key)
- parameters.append(_required_string_parameter_row(key))
-
-
def _coerce_positive_int(value: Any) -> int | None:
try:
parsed = int(value)
@@ -7215,7 +7164,6 @@ def _scout_interaction_for_synthesized_parameter(
def _reconcile_synthesized_parameters(
*,
- ctx: AgentContext | None = None,
parsed: dict[str, Any],
code_block: dict[str, Any],
submitted_code: str,
@@ -7251,9 +7199,6 @@ def _reconcile_synthesized_parameters(
for interaction in scout_trajectory
if str(interaction.get("tool_name") or "") == "type_text"
]
- binding_steering_exhausted = (
- _synthesized_parameter_binding_reject_count(ctx) >= _MAX_OUTPUT_CONTRACT_STEERING_REJECTS
- )
def add_binding_violation(key: str, message: str) -> None:
nonlocal repair_context
@@ -7296,13 +7241,10 @@ def _reconcile_synthesized_parameters(
)
if selector_join_alias is not None:
if selector_join_alias in used_selector_join_aliases:
- if binding_steering_exhausted:
- _impose_required_string_parameter(parameters, key)
- else:
- add_binding_violation(
- key,
- f"Unable to bind synthesized parameter `{key}`: authored selector-join alias is reused by another synthesized input.",
- )
+ add_binding_violation(
+ key,
+ f"Unable to bind synthesized parameter `{key}`: authored selector-join alias is reused by another synthesized input.",
+ )
continue
aliases[key] = selector_join_alias
used_selector_join_aliases.add(selector_join_alias)
@@ -7341,24 +7283,18 @@ def _reconcile_synthesized_parameters(
_drop_parameter_key(parameters, key)
continue
if _is_credential_parameter(existing):
- if binding_steering_exhausted:
- _impose_required_string_parameter(parameters, key)
- else:
- add_binding_violation(
- key,
- f"Unable to bind synthesized parameter `{key}`: submitted parameter is credential-typed.",
- )
+ add_binding_violation(
+ key,
+ f"Unable to bind synthesized parameter `{key}`: submitted parameter is credential-typed.",
+ )
elif synthesized_default:
existing_default = _string_parameter_default_value(existing)
if existing_default != synthesized_default:
- if binding_steering_exhausted:
- _impose_required_string_parameter(parameters, key)
- else:
- add_binding_violation(
- key,
- f"Unable to bind synthesized parameter `{key}`: "
- "submitted parameter default does not match the scout record.",
- )
+ add_binding_violation(
+ key,
+ f"Unable to bind synthesized parameter `{key}`: "
+ "submitted parameter default does not match the scout record.",
+ )
continue
if synthesized_default:
@@ -7384,27 +7320,21 @@ def _reconcile_synthesized_parameters(
continue
if len(non_credential_synthesized) != 1:
- if binding_steering_exhausted:
- parameters.append(_required_string_parameter_row(key))
- else:
- add_binding_violation(
- key,
- f"Unable to bind synthesized parameter `{key}`: missing submitted workflow parameter and literal binding is ambiguous.",
- )
+ add_binding_violation(
+ key,
+ f"Unable to bind synthesized parameter `{key}`: missing submitted workflow parameter and literal binding is ambiguous.",
+ )
continue
typed_length = typed_length or scout_typed_length or (typed_lengths[0] if len(typed_lengths) == 1 else None)
direct_fill_usage = _submitted_direct_fill_type_usage(submitted_code, key)
if direct_fill_usage.matched and direct_fill_usage.mismatched:
- if binding_steering_exhausted:
- parameters.append(_required_string_parameter_row(key))
- else:
- add_binding_violation(
- key,
- f"Unable to bind synthesized parameter `{key}`: submitted code mixes direct fills using `{key}` "
- "with other browser-locator fill/type values. Use the synthesized parameter for every scout-input "
- "fill in the code block, or declare explicit workflow parameters/defaults for the other filled values.",
- )
+ add_binding_violation(
+ key,
+ f"Unable to bind synthesized parameter `{key}`: submitted code mixes direct fills using `{key}` "
+ "with other browser-locator fill/type values. Use the synthesized parameter for every scout-input "
+ "fill in the code block, or declare explicit workflow parameters/defaults for the other filled values.",
+ )
continue
if direct_fill_usage.matched:
parameters.append(_required_string_parameter_row(key))
@@ -7412,10 +7342,7 @@ def _reconcile_synthesized_parameters(
literal, error = _safe_singleton_literal_for_parameter(submitted_code, key, typed_length)
if error:
- if binding_steering_exhausted:
- parameters.append(_required_string_parameter_row(key))
- else:
- add_binding_violation(key, error)
+ add_binding_violation(key, error)
continue
parameters.append(
{
@@ -7659,7 +7586,6 @@ def _maybe_impose_synthesized_code_block(workflow_yaml: str, ctx: AgentContext)
)
parameter_reconciliation = _reconcile_synthesized_parameters(
- ctx=ctx,
parsed=parsed,
code_block=code_block,
submitted_code=submitted_code,
@@ -8906,6 +8832,7 @@ async def _update_workflow(
*,
allow_missing_credentials: bool | None = None,
allow_static_output_uncertainty: bool = False,
+ formation_prepared: bool = False,
) -> dict[str, Any]:
def reject(
*,
@@ -9003,6 +8930,7 @@ async def _update_workflow(
if _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER:
workflow_yaml = _adopt_exact_declared_parameter_keys_for_unresolved_names(workflow_yaml)
params["workflow_yaml"] = workflow_yaml
+ metadata_scrubbed_by_imposition = False
if (
imposition.selected_extraction_metadata_disposition == "browser_spine_replaced_metadata_stale"
and imposition.scrubbed_selected_metadata_label
@@ -9013,6 +8941,7 @@ async def _update_workflow(
)
params["code_artifact_metadata"] = scrubbed_metadata
ctx.raw_code_artifact_metadata = scrubbed_metadata
+ metadata_scrubbed_by_imposition = True
parameter_contract_error = _code_block_parameter_contract_error(workflow_yaml)
if parameter_contract_error is not None:
return reject(
@@ -9020,20 +8949,28 @@ async def _update_workflow(
user_facing_summary=_compiled_authoring_user_summary(),
data=_code_repair_progress_data(),
)
- workflow_yaml, imposed_metadata, envelope_imposed = _impose_output_contract_envelope_after_steering(
- ctx,
- workflow_yaml,
- params.get("code_artifact_metadata"),
+ workflow_yaml, imposed_metadata, envelope_imposed = (
+ (workflow_yaml, params.get("code_artifact_metadata"), False)
+ if formation_prepared and imposition.substitutions is None
+ else _impose_output_contract_envelope_after_steering(
+ ctx,
+ workflow_yaml,
+ params.get("code_artifact_metadata"),
+ )
)
if envelope_imposed:
params["workflow_yaml"] = workflow_yaml
params["code_artifact_metadata"] = imposed_metadata
params["raw_code_artifact_metadata"] = imposed_metadata
ctx.raw_code_artifact_metadata = imposed_metadata
- scaffolded_metadata, scaffold_applied = _scaffold_metadata_contract_for_update(
- ctx,
- workflow_yaml,
- params.get("code_artifact_metadata"),
+ scaffolded_metadata, scaffold_applied = (
+ (params.get("code_artifact_metadata"), False)
+ if formation_prepared and not metadata_scrubbed_by_imposition
+ else _scaffold_metadata_contract_for_update(
+ ctx,
+ workflow_yaml,
+ params.get("code_artifact_metadata"),
+ )
)
if scaffold_applied:
params["code_artifact_metadata"] = scaffolded_metadata
@@ -9075,22 +9012,45 @@ async def _update_workflow(
authored_structural_fingerprint=authored_fingerprint,
workflow_yaml=workflow_yaml,
)
- reject_result = _output_contract_reject_result(
- output_contract_evaluation,
- payload=payload,
- tool_name="update_workflow",
- )
- return reject(
- error=str(reject_result["error"]),
- user_facing_summary=str(reject_result.get("user_facing_summary") or _compiled_authoring_user_summary()),
- data=reject_result.get("data") if isinstance(reject_result.get("data"), dict) else None,
- repair_context=output_contract_evaluation.repair_context,
- record_repair_context_outcome=False,
- )
- missing_metadata_error = _missing_code_artifact_metadata_error(
- workflow_yaml,
- ctx,
- params.get("code_artifact_metadata"),
+ if allow_static_output_uncertainty and _output_contract_advisory_granted(
+ ctx, output_contract_evaluation.canonical_signature
+ ):
+ _arm_pending_run_evidence(
+ ctx,
+ output_contract_evaluation.canonical_signature,
+ set(output_contract_evaluation.observation_paths),
+ )
+ else:
+ reject_result = _output_contract_reject_result(
+ output_contract_evaluation,
+ payload=payload,
+ tool_name="update_workflow",
+ )
+ return reject(
+ error=str(reject_result["error"]),
+ user_facing_summary=str(
+ reject_result.get("user_facing_summary") or _compiled_authoring_user_summary()
+ ),
+ data=reject_result.get("data") if isinstance(reject_result.get("data"), dict) else None,
+ repair_context=output_contract_evaluation.repair_context,
+ record_repair_context_outcome=False,
+ )
+ typed_output_owner_contract_complete = (
+ formation_prepared
+ and output_contract_evaluation is not None
+ and not output_contract_evaluation.has_deficiencies
+ )
+ # update_and_run_blocks already resolved the exact output owner and validated its typed
+ # contract. Do not let the legacy output-intent scan reclassify a non-owner browser block
+ # as another output owner. Save-only update_workflow retains the conservative label gate.
+ missing_metadata_error = (
+ None
+ if typed_output_owner_contract_complete
+ else _missing_code_artifact_metadata_error(
+ workflow_yaml,
+ ctx,
+ params.get("code_artifact_metadata"),
+ )
)
if missing_metadata_error is not None:
missing_labels = _missing_code_artifact_metadata_labels(
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index f9fd3a836..dbcb96807 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -2098,6 +2098,7 @@ workflow_definition:
async def fake_update_workflow(payload, _ctx, **_kwargs):
captured_metadata.extend(payload["code_artifact_metadata"])
+ captured_metadata.append({"formation_prepared": _kwargs["formation_prepared"]})
return {"ok": False, "error": "sentinel update reached", "data": {"from_update": True}}
monkeypatch.setattr(tools_module, "_request_policy_allows_update_and_skip_run", lambda *args: False)
@@ -2122,9 +2123,10 @@ workflow_definition:
"output.flags",
"output.record_id",
]
+ assert captured_metadata[-1]["formation_prepared"] is True
@pytest.mark.asyncio
- async def test_update_and_run_blocks_post_steering_static_return_gap_reaches_run(self, monkeypatch) -> None:
+ async def test_update_and_run_blocks_typed_advisory_static_return_gap_reaches_run(self, monkeypatch) -> None:
workflow_yaml = """
title: Test workflow
workflow_definition:
@@ -2158,7 +2160,7 @@ workflow_definition:
reason_code="requested_output_contract_missing_output_coverage",
required_paths=required_paths,
)
- ctx.output_contract_reject_count_by_signature = {signature: 2}
+ workflow_update_module._grant_output_contract_advisory_run(ctx, signature)
captured: dict[str, object] = {}
async def fake_update_workflow(payload, update_ctx, **_kwargs):
@@ -2231,7 +2233,11 @@ workflow_definition:
captured: dict[str, str | bool] = {}
async def fake_update_workflow(
- payload, ctx, allow_missing_credentials=False, allow_static_output_uncertainty=False
+ payload,
+ ctx,
+ allow_missing_credentials=False,
+ allow_static_output_uncertainty=False,
+ formation_prepared=False,
):
captured["workflow_yaml"] = payload["workflow_yaml"]
ctx.workflow_yaml = payload["workflow_yaml"]
diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.py
index 55a5858e1..0a455c029 100644
--- a/tests/unit/test_copilot_code_block_persist_seam.py
+++ b/tests/unit/test_copilot_code_block_persist_seam.py
@@ -9,7 +9,7 @@ import ast
import json
import textwrap
from types import SimpleNamespace
-from unittest.mock import AsyncMock
+from unittest.mock import AsyncMock, Mock
import pytest
import yaml
@@ -1519,11 +1519,14 @@ class TestCodeRepairProgressClassification:
_stub_successful_update(monkeypatch)
ctx = _code_only_ctx()
label = "submit_form_and_extract_confirmation"
+ schema = workflow_update_module._schema_template_text_for_required_paths({"output.confirmation_number"})
ctx.code_artifact_metadata = {
label: {
"block_label": label,
- "claimed_outcomes": [{"goal_value_paths": ["output.confirmation_number"]}],
- "terminal_verifier_expectations": [{"goal_value_paths": ["output.confirmation_number"]}],
+ "claimed_outcomes": [{"goal_value_paths": ["output.confirmation_number"], "extraction_schema": schema}],
+ "terminal_verifier_expectations": [
+ {"goal_value_paths": ["output.confirmation_number"], "extraction_schema": schema}
+ ],
}
}
ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
@@ -1559,7 +1562,7 @@ class TestCodeRepairProgressClassification:
assert result["ok"] is True
assert ctx.workflow_yaml is not None
assert "confirmation_number" in ctx.workflow_yaml
- assert "extraction_schema" in ctx.workflow_yaml
+ assert ctx.code_artifact_metadata[label]["claimed_outcomes"][0]["extraction_schema"] == schema
assert ctx.latest_recorded_build_test_outcome is not None
assert ctx.latest_recorded_build_test_outcome.reason_code == "outcome_not_demonstrated"
@@ -2944,7 +2947,7 @@ class TestCodeRepairProgressClassification:
)
assert update_result["data"]["satisfying_templates"] == run_result["data"]["satisfying_templates"]
- def test_static_return_uncertainty_is_run_preflight_advisory_only(self) -> None:
+ def test_static_return_uncertainty_requires_typed_advisory_grant(self) -> None:
ctx = _code_only_ctx()
ctx.last_code_authoring_repair_context = CodeAuthoringRepairContext(
block_label="extract_entry_output",
@@ -2988,9 +2991,19 @@ class TestCodeRepairProgressClassification:
assert update_eval.missing_return_paths == ["output.record_id"]
assert update_eval.can_attempt_run is False
assert run_eval is not None
- assert run_eval.missing_return_paths == []
- assert run_eval.payload["static_return_advisory_paths"] == ["output.record_id"]
- assert run_eval.can_attempt_run is True
+ assert run_eval.missing_return_paths == ["output.record_id"]
+ assert run_eval.can_attempt_run is False
+
+ workflow_update_module._grant_output_contract_advisory_run(ctx, run_eval.canonical_signature)
+ granted_eval = workflow_update_module._evaluate_output_contract_for_code_block(
+ ctx, workflow_yaml, metadata, allow_static_return_advisory=True
+ )
+
+ assert granted_eval is not None
+ assert granted_eval.missing_return_paths == []
+ assert granted_eval.payload["static_return_advisory_paths"] == ["output.record_id"]
+ assert granted_eval.payload["actuated_static_return_advisory"] is True
+ assert granted_eval.can_attempt_run is True
def test_complete_metadata_without_code_return_is_run_preflight_advisory(self) -> None:
ctx = _code_only_ctx()
@@ -3041,7 +3054,7 @@ class TestCodeRepairProgressClassification:
reason_code="requested_output_contract_missing_output_coverage",
required_paths=required_paths,
)
- ctx.output_contract_reject_count_by_signature = {signature: 2}
+ workflow_update_module._grant_output_contract_advisory_run(ctx, signature)
evaluation = workflow_update_module._evaluate_output_contract_for_code_block(
ctx, workflow_yaml, metadata, allow_static_return_advisory=True
@@ -3052,11 +3065,11 @@ class TestCodeRepairProgressClassification:
assert evaluation.can_attempt_run is True
assert evaluation.missing_return_paths == []
assert evaluation.payload["static_return_advisory_paths"] == sorted(required_paths)
- assert evaluation.payload["post_steering_static_return_advisory"] is True
+ assert evaluation.payload["actuated_static_return_advisory"] is True
assert result is None
@pytest.mark.asyncio
- async def test_save_only_update_rejects_post_steering_static_return_gap(
+ async def test_save_only_update_rejects_static_return_gap_without_typed_grant(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_stub_successful_update(monkeypatch)
@@ -3082,15 +3095,6 @@ class TestCodeRepairProgressClassification:
"""
)
ctx.turn_id = "save-only-static-return-gap"
- signature = workflow_update_module._output_contract_signature(
- ctx=ctx,
- workflow_yaml=workflow_yaml,
- source="requested_output_contract",
- reason_code="requested_output_contract_missing_output_coverage",
- required_paths={"output.record_id"},
- )
- ctx.output_contract_reject_count_by_signature = {signature: 2}
-
result = await _update_workflow(
{"workflow_yaml": workflow_yaml},
ctx,
@@ -3103,7 +3107,7 @@ class TestCodeRepairProgressClassification:
assert result["data"]["static_return_advisory_paths"] == []
@pytest.mark.asyncio
- async def test_run_path_allows_post_steering_declared_output_return_shape_gap(
+ async def test_run_path_allows_typed_advisory_declared_output_return_shape_gap(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_stub_successful_update(monkeypatch)
@@ -3136,7 +3140,7 @@ class TestCodeRepairProgressClassification:
reason_code="requested_output_contract_missing_output_coverage",
required_paths={"output.record_id"},
)
- ctx.output_contract_reject_count_by_signature = {signature: 2}
+ workflow_update_module._grant_output_contract_advisory_run(ctx, signature)
result = await _update_workflow(
{"workflow_yaml": workflow_yaml},
@@ -3621,7 +3625,7 @@ class TestCodeRepairProgressClassification:
assert first_scope_key != other_path_key
@pytest.mark.asyncio
- async def test_output_contract_after_two_steering_cycles_imposes_keyed_return_envelope(
+ async def test_output_contract_first_contact_imposes_keyed_return_envelope(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_stub_successful_update(monkeypatch)
@@ -3657,13 +3661,9 @@ class TestCodeRepairProgressClassification:
"""
)
- first = await _update_workflow({"workflow_yaml": workflow_yaml}, ctx, allow_missing_credentials=True)
- second = await _update_workflow({"workflow_yaml": workflow_yaml}, ctx, allow_missing_credentials=True)
- imposed = await _update_workflow({"workflow_yaml": workflow_yaml}, ctx, allow_missing_credentials=True)
+ result = await _update_workflow({"workflow_yaml": workflow_yaml}, ctx, allow_missing_credentials=True)
- assert first["ok"] is False
- assert second["ok"] is False
- assert imposed["ok"] is True
+ assert result["ok"] is True
parsed = parse_workflow_yaml(ctx.workflow_yaml)
assert isinstance(parsed, dict)
code = str(_single_code_block(parsed)["code"])
@@ -3678,7 +3678,7 @@ class TestCodeRepairProgressClassification:
)
@pytest.mark.asyncio
- async def test_requested_output_contract_omits_repair_context_when_output_target_is_ambiguous(
+ async def test_requested_output_contract_routes_ambiguous_output_target_to_typed_repair(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_stub_successful_update(monkeypatch)
@@ -3722,8 +3722,9 @@ class TestCodeRepairProgressClassification:
assert result["ok"] is False
assert "requested output contract" in result["error"]
- assert "authoring_repair_context" not in result["data"]
- assert ctx.last_code_authoring_repair_context is None
+ assert result["data"]["authoring_repair_context"]["reason_code"] == "output_owner_ambiguous"
+ assert ctx.last_code_authoring_repair_context is not None
+ assert ctx.last_code_authoring_repair_context.reason_code == "output_owner_ambiguous"
outcome = ctx.latest_recorded_build_test_outcome
assert outcome is not None
assert outcome.reason_code == "metadata_reject"
@@ -3980,7 +3981,8 @@ class TestCodeRepairProgressClassification:
assert results[0]["data"]["reason_code"] == "metadata_contract_required_before_run"
assert results[0]["data"]["output_contract_reason_code"] == "output_contract_required"
assert results[0]["data"]["output_contract_reject_count"] == 1
- assert results[-1] is None
+ assert results[-1] is not None
+ assert results[-1]["data"]["output_contract_reject_count"] == 4
assert all(
result is None or result["data"]["reason_code"] != "output_contract_reject_budget_exhausted"
for result in results
@@ -5357,7 +5359,7 @@ class TestCodeRepairProgressClassification:
assert evaluation.payload["output_owner_labels"] == ["new_owner"]
@pytest.mark.asyncio
- async def test_runtime_output_repair_facts_trigger_one_envelope_attempt_before_budget(
+ async def test_runtime_output_repair_facts_trigger_first_contact_envelope(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_stub_successful_update(monkeypatch)
@@ -5519,6 +5521,79 @@ class TestCodeRepairProgressClassification:
labels = [block.get("label") for block in workflow_blocks(parse_workflow_yaml(ctx.workflow_yaml))]
assert labels == ["enter_filters", "choose_record", "extract_record"]
+ @pytest.mark.asyncio
+ async def test_formation_prepared_p4_two_block_output_owner_ignores_generic_non_owner_label_reject(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ SimpleNamespace(
+ id="requested_npi",
+ output_path="output.npi",
+ level="run",
+ method_mandated=False,
+ kind="outcome",
+ )
+ ]
+ )
+ workflow_yaml = _yaml(
+ """
+ title: Directory lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: search_directory
+ prompt: Search the directory and extract provider result data.
+ code: |
+ await page.locator("#npi").fill("1234567890")
+ await page.locator("#search_by").click()
+ - block_type: code
+ label: extract_directory_result
+ prompt: Read the provider card and return the NPI.
+ code: |
+ await page.locator("#coastalCard").wait_for(state="visible")
+ return {"output": {"npi": "1234567890"}}
+ """
+ )
+ schema = '{"type":"object","properties":{"output":{"type":"object","properties":{"npi":{"type":"string"}}}}}'
+ metadata = [
+ {
+ "block_label": "extract_directory_result",
+ "claimed_outcomes": [{"goal_value_paths": ["output.npi"], "extraction_schema": schema}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["output.npi"], "extraction_schema": schema}],
+ }
+ ]
+
+ save_only = await _update_workflow(
+ {"workflow_yaml": workflow_yaml, "code_artifact_metadata": metadata},
+ ctx,
+ allow_missing_credentials=True,
+ )
+
+ assert save_only["ok"] is False
+ assert "search_directory" in save_only["error"]
+
+ run_ctx = _code_only_ctx()
+ run_ctx.request_policy = ctx.request_policy
+ prepared_run = await _update_workflow(
+ {"workflow_yaml": workflow_yaml, "code_artifact_metadata": metadata},
+ run_ctx,
+ allow_missing_credentials=True,
+ allow_static_output_uncertainty=True,
+ formation_prepared=True,
+ )
+
+ assert prepared_run["ok"] is True
+ assert run_ctx.latest_recorded_build_test_outcome is None
+ assert workflow_update_module._artifact_declares_goal_values(
+ run_ctx.code_artifact_metadata["extract_directory_result"]
+ )
+ assert not workflow_update_module._artifact_declares_goal_values(
+ run_ctx.code_artifact_metadata["search_directory"]
+ )
+
@pytest.mark.asyncio
async def test_separated_spine_shape_rejects_collapsed_output_owner(self, monkeypatch: pytest.MonkeyPatch) -> None:
_stub_successful_update(monkeypatch)
@@ -6764,10 +6839,6 @@ def _spine_actuation_ctx() -> CopilotContext:
)
]
)
- signature = workflow_update_module._stable_output_contract_key("turn:t-spine", {"output.record_id"})
- ctx.output_contract_reject_count_by_signature = {
- signature: workflow_update_module._MAX_OUTPUT_CONTRACT_STEERING_REJECTS
- }
return ctx
@@ -6952,7 +7023,13 @@ class TestSeparatedSpineViolationActuation:
"_setup = 1\n" + _SPINE_SYNTH_CODE + '\nreturn {"output": {"record_id": "X"}}'
)
- result = workflow_update_module._metadata_contract_run_preflight_reject(ctx, workflow_yaml, [])
+ prepared_yaml, prepared_metadata, _ = workflow_update_module._impose_output_contract_envelope_after_steering(
+ ctx, workflow_yaml, []
+ )
+ prepared_metadata, _ = workflow_update_module._scaffold_metadata_contract_for_update(
+ ctx, prepared_yaml, prepared_metadata
+ )
+ result = workflow_update_module._metadata_contract_run_preflight_reject(ctx, prepared_yaml, prepared_metadata)
assert result is not None
assert result["ok"] is False
@@ -7107,7 +7184,13 @@ class TestAmbiguousOutputOwnerActuation:
ctx = _spine_actuation_ctx()
workflow_yaml = _dual_output_owner_yaml()
- result = workflow_update_module._metadata_contract_run_preflight_reject(ctx, workflow_yaml, [])
+ prepared_yaml, prepared_metadata, _ = workflow_update_module._impose_output_contract_envelope_after_steering(
+ ctx, workflow_yaml, []
+ )
+ prepared_metadata, _ = workflow_update_module._scaffold_metadata_contract_for_update(
+ ctx, prepared_yaml, prepared_metadata
+ )
+ result = workflow_update_module._metadata_contract_run_preflight_reject(ctx, prepared_yaml, prepared_metadata)
assert result is not None
assert result["ok"] is False
@@ -8708,96 +8791,22 @@ class TestCompiledAuthoringImposition:
assert ctx.workflow_yaml == ""
@pytest.mark.asyncio
- async def test_parameter_binding_after_two_steering_cycles_imposes_synthesized_binding(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = self._provider_search_ctx()
- first = await _update_workflow({"workflow_yaml": _SUBMITTED_UNKNOWN_COMPUTED_LITERAL_YAML}, ctx)
- second = await _update_workflow({"workflow_yaml": _SUBMITTED_UNKNOWN_COMPUTED_LITERAL_YAML}, ctx)
- imposed = await _update_workflow({"workflow_yaml": _SUBMITTED_UNKNOWN_COMPUTED_LITERAL_YAML}, ctx)
-
- assert first["ok"] is False
- assert second["ok"] is False
- assert imposed["ok"] is True
- parsed = parse_workflow_yaml(ctx.workflow_yaml)
- assert isinstance(parsed, dict)
- block = _single_code_block(parsed)
- assert block["parameter_keys"] == ["provider_name"]
- assert 'await page.locator("#provInput").fill(str(provider_name))' in block["code"]
-
- @pytest.mark.asyncio
- async def test_parameter_binding_after_two_steering_cycles_imposes_for_mixed_fill(
- self, monkeypatch: pytest.MonkeyPatch
+ @pytest.mark.parametrize(
+ "workflow_yaml",
+ [_SUBMITTED_UNKNOWN_COMPUTED_LITERAL_YAML, _SUBMITTED_MIXED_LOCATOR_FILL_COMPUTED_LITERAL_YAML],
+ )
+ async def test_ambiguous_parameter_binding_rejects_without_count_authority(
+ self, monkeypatch: pytest.MonkeyPatch, workflow_yaml: str
) -> None:
_stub_successful_update(monkeypatch)
ctx = self._provider_search_ctx()
+ ctx.output_contract_reject_count_by_signature = {"unrelated": 99}
- first = await _update_workflow({"workflow_yaml": _SUBMITTED_MIXED_LOCATOR_FILL_COMPUTED_LITERAL_YAML}, ctx)
- second = await _update_workflow({"workflow_yaml": _SUBMITTED_MIXED_LOCATOR_FILL_COMPUTED_LITERAL_YAML}, ctx)
- imposed = await _update_workflow({"workflow_yaml": _SUBMITTED_MIXED_LOCATOR_FILL_COMPUTED_LITERAL_YAML}, ctx)
-
- assert first["ok"] is False
- assert second["ok"] is False
- assert imposed["ok"] is True
- parsed = parse_workflow_yaml(ctx.workflow_yaml)
- assert isinstance(parsed, dict)
- block = _single_code_block(parsed)
- assert block["parameter_keys"] == ["provider_name"]
- assert parsed["workflow_definition"]["parameters"] == [
- {
- "parameter_type": "workflow",
- "workflow_parameter_type": "string",
- "key": "provider_name",
- }
- ]
-
- def test_parameter_binding_uses_canonical_output_contract_budget(self) -> None:
- ctx = self._provider_search_ctx()
- ctx.turn_id = "parameter-binding-contract-budget"
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- SimpleNamespace(
- id="requested_value",
- output_path="output.record_id",
- level="run",
- method_mandated=False,
- kind="outcome",
- )
- ]
- )
- signature = workflow_update_module._output_contract_signature(
- ctx=ctx,
- workflow_yaml="title: First\nworkflow_definition:\n blocks: []\n",
- source="metadata_reject",
- reason_code="metadata_reject",
- required_paths={"output.record_id"},
- )
- ctx.output_contract_reject_count_by_signature = {signature: 2}
- parsed = parse_workflow_yaml(_SUBMITTED_UNKNOWN_COMPUTED_LITERAL_YAML)
- assert isinstance(parsed, dict)
- block = _single_code_block(parsed)
- synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
- assert synthesized is not None
-
- reconciliation = workflow_update_module._reconcile_synthesized_parameters(
- ctx=ctx,
- parsed=parsed,
- code_block=block,
- submitted_code=str(block.get("code") or ""),
- synthesized_parameters=synthesized.parameters,
- scout_trajectory=ctx.scout_trajectory,
- )
+ result = await _update_workflow({"workflow_yaml": workflow_yaml}, ctx)
- assert reconciliation.violations == []
- assert reconciliation.parameter_keys == ["provider_name"]
- assert parsed["workflow_definition"]["parameters"] == [
- {
- "parameter_type": "workflow",
- "workflow_parameter_type": "string",
- "key": "provider_name",
- }
- ]
+ assert result["ok"] is False
+ assert result["data"]["authoring_repair_context"]["reason_code"] == ("synthesized_parameter_binding_ambiguous")
+ assert ctx.workflow_yaml == ""
@pytest.mark.asyncio
async def test_synthesized_parameter_repair_context_uses_safe_selector_atom(
@@ -11986,6 +11995,94 @@ class TestWholeTrajectoryImposition:
artifact = ctx.code_artifact_metadata["quote_flow"]
assert not workflow_update_module._artifact_declares_goal_values(artifact)
+ @pytest.mark.asyncio
+ async def test_formation_prepared_directory_run_rescaffolds_metadata_after_imposition_scrub(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ scaffold = Mock(wraps=workflow_update_module._scaffold_metadata_contract_for_update)
+ envelope = Mock(wraps=workflow_update_module._impose_output_contract_envelope_after_steering)
+ monkeypatch.setattr(workflow_update_module, "_scaffold_metadata_contract_for_update", scaffold)
+ monkeypatch.setattr(workflow_update_module, "_impose_output_contract_envelope_after_steering", envelope)
+ ctx = _code_only_ctx()
+ _enable_imposition(ctx)
+ ctx.scout_trajectory = [
+ {
+ "tool_name": "type_text",
+ "selector": "#npi",
+ "source_url": "https://example.com/directory",
+ "typed_length": 10,
+ "typed_value": "1234567890",
+ "role": "textbox",
+ "accessible_name": "NPI",
+ "trajectory_index": 0,
+ },
+ {
+ "tool_name": "click",
+ "selector": "#search_by",
+ "source_url": "https://example.com/directory",
+ "trajectory_index": 1,
+ },
+ ]
+ scouting_module._mint_current_loaded_result_source(
+ ctx,
+ {
+ "forms": [],
+ "navigation_targets": [],
+ "result_containers": [
+ {"selector": "#coastalCard", "text": "Provider record is visible", "row_count": 1}
+ ],
+ },
+ url="https://example.com/directory",
+ )
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ SimpleNamespace(
+ id="requested_npi",
+ output_path="output.npi",
+ level="run",
+ method_mandated=False,
+ kind="outcome",
+ )
+ ]
+ )
+ submitted = _yaml(
+ """
+ title: Directory lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: directory_flow
+ code: |
+ await page.locator("#npi").fill(str(npi))
+ await page.locator("#search_by").click()
+ await page.locator("#coastalCard").wait_for(state="visible")
+ """
+ )
+ metadata = [
+ {
+ "block_label": "directory_flow",
+ "claimed_outcomes": [{"goal_value_paths": ["output.npi"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["output.npi"]}],
+ }
+ ]
+
+ result = await _update_workflow(
+ {"workflow_yaml": submitted, "code_artifact_metadata": metadata},
+ ctx,
+ allow_static_output_uncertainty=True,
+ formation_prepared=True,
+ )
+
+ assert result["ok"] is False
+ assert envelope.call_count == 1
+ assert scaffold.call_count == 1
+ artifact = workflow_update_module._metadata_item_for_block_label(
+ ctx.raw_code_artifact_metadata, "directory_flow"
+ )
+ assert artifact is not None
+ assert not workflow_update_module._artifact_declares_goal_values(artifact)
+
@pytest.mark.asyncio
async def test_metadata_selected_page_goto_extra_is_discarded(self, monkeypatch: pytest.MonkeyPatch) -> None:
_stub_successful_update(monkeypatch)
@@ -12668,14 +12765,8 @@ def _declaration_stamp_ctx() -> CopilotContext:
class TestDeclarationContractStamp:
- def test_zero_output_block_persists_declared_blocker_at_ceiling(self) -> None:
+ def test_zero_output_block_persists_declared_blocker_on_first_contact(self) -> None:
ctx = _declaration_stamp_ctx()
- signature = workflow_update_module._stable_output_contract_key(
- "turn:t-decl", {"output.record_id", "output.blocker"}
- )
- ctx.output_contract_reject_count_by_signature = {
- signature: workflow_update_module._MAX_OUTPUT_CONTRACT_STEERING_REJECTS
- }
workflow_yaml = _collapsed_spine_yaml('await page.click("#submit")')
new_yaml, _metadata, applied = workflow_update_module._impose_output_contract_envelope_after_steering(
@@ -12689,7 +12780,7 @@ class TestDeclarationContractStamp:
assert "output.record_id" not in produced
assert '"blocker": None' in code
- def test_stamp_applies_before_steering_reject_wait(self) -> None:
+ def test_stamp_applies_before_output_contract_actuation(self) -> None:
ctx = _declaration_stamp_ctx()
workflow_yaml = _collapsed_spine_yaml('await page.click("#submit")\nreturn {}')
diff --git a/tests/unit/test_copilot_output_contract_actuation.py b/tests/unit/test_copilot_output_contract_actuation.py
index c5ec5de65..b1c0743cd 100644
--- a/tests/unit/test_copilot_output_contract_actuation.py
+++ b/tests/unit/test_copilot_output_contract_actuation.py
@@ -33,6 +33,7 @@ from skyvern.forge.sdk.copilot.output_contracts import (
resolve_output_contract_actuation,
)
from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, RequestPolicy
+from skyvern.forge.sdk.copilot.result_evidence import loaded_result_composition_evidence_from_page
from skyvern.forge.sdk.copilot.runtime import (
output_contract_ladder_unresolved,
record_author_time_gate_ablation_event,
@@ -419,6 +420,11 @@ def test_observed_values_suppresses_no_source_terminal() -> None:
has_source = OutputContractActuationEvidence(False, True, True, True, False, OutputContractAdvisoryState.UNUSED)
actuation = resolve_output_contract_actuation(family=OutputContractBailFamily.STATIC_RETURN, evidence=has_source)
assert actuation.kind != OutputContractActuationKind.BLOCKED_TERMINAL
+ loaded_source = OutputContractActuationEvidence(
+ False, True, False, True, False, OutputContractAdvisoryState.UNUSED, loaded_result_source_producible=True
+ )
+ actuation = resolve_output_contract_actuation(family=OutputContractBailFamily.STATIC_RETURN, evidence=loaded_source)
+ assert actuation.kind != OutputContractActuationKind.BLOCKED_TERMINAL
def test_classifier_maps_every_emitted_blocker_string() -> None:
@@ -516,6 +522,7 @@ def _advisory_ctx() -> SimpleNamespace:
output_contract_reject_count_by_signature={},
output_contract_imposed_since_last_reject_by_signature={},
output_contract_armed_directive_fingerprint_by_signature={},
+ output_contract_output_owner_directive_candidates_by_signature={},
output_contract_actuation_by_signature={},
output_contract_actuation_count_by_signature={},
output_contract_declick_attempted_by_signature={},
@@ -531,6 +538,8 @@ def _advisory_ctx() -> SimpleNamespace:
scouted_output_covered_paths=set(),
composition_page_evidence=None,
recorded_outcome_binding_constraint=None,
+ latest_evaluate_result_composition_steer=None,
+ latest_evaluate_result_composition_signature=None,
)
@@ -1091,6 +1100,38 @@ def test_reject_seam_adjudication_grants_advisory_within_caps_when_imposition_ea
assert ctx.turn_halt is None
+def test_same_fresh_signature_emits_one_reject_before_typed_advisory(monkeypatch: pytest.MonkeyPatch) -> None:
+ ctx = _ladder_ctx()
+ signature = "sig_first_contact_example"
+ evaluation = _make_evaluation(signature, block_label="collect")
+ monkeypatch.setattr(wu, "_record_code_authoring_guardrail_reject", lambda _ctx: None)
+
+ first = wu._record_output_contract_reject(
+ ctx,
+ evaluation,
+ summary="First typed deficiency.",
+ authored_structural_fingerprint="fp_first_contact",
+ workflow_yaml=_PAGE_READ_YAML,
+ )
+ first_outcome = ctx.latest_recorded_build_test_outcome
+ ctx.output_contract_bail_actuated_this_call = False
+ second = wu._record_output_contract_reject(
+ ctx,
+ evaluation,
+ summary="Same typed deficiency.",
+ authored_structural_fingerprint="fp_first_contact",
+ workflow_yaml=_PAGE_READ_YAML,
+ )
+
+ assert first["output_contract_actuation"] == OutputContractActuationKind.STRUCTURE_DIRECTIVE.value
+ assert first_outcome is not None
+ assert first_outcome.phase == "author_time_reject"
+ assert second["output_contract_actuation"] == OutputContractActuationKind.ADVISORY_RUN.value
+ assert ctx.latest_recorded_build_test_outcome is None
+ assert [entry["phase"] for entry in ctx.recorded_build_test_outcome_history] == ["author_time_reject"]
+ assert ctx.output_contract_actuation_by_signature[signature] == OutputContractAdvisoryState.GRANTED
+
+
def test_reject_seam_metadata_required_reaches_advisory_before_no_source_terminal() -> None:
ctx = _ladder_ctx()
signature = "sig_metadata_required"
@@ -1368,6 +1409,55 @@ def test_advisory_grant_downgraded_to_directive_when_run_authority_forbids_dispa
assert signature not in ctx.output_contract_actuation_by_signature
+def test_loaded_result_carrier_is_selector_bound_and_claimed_by_one_contract_signature(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ ctx = _advisory_ctx()
+ events: list[dict[str, object]] = []
+
+ def capture_event(_ctx: object, **kwargs: object) -> bool:
+ events.append(kwargs)
+ return False
+
+ monkeypatch.setattr(wu, "record_author_time_gate_ablation_event", capture_event)
+ ctx.latest_evaluate_result_composition_steer = loaded_result_composition_evidence_from_page(
+ {"result_containers": [{"selector": "#results", "row_count": 1, "sample_rows": ["Example customer record"]}]},
+ source_tool="evaluate",
+ source_url="https://example.com/results",
+ )
+
+ first = wu._actuate_output_contract_bail(
+ ctx,
+ blockers=["static_return_envelope_unavailable"],
+ target_code='await page.locator("#results").click()',
+ required_paths={"output.record_id"},
+ signature="sig-a",
+ current_fingerprint="fp-a",
+ )
+
+ assert ctx.latest_evaluate_result_composition_signature == "sig-a"
+
+ ctx.output_contract_bail_actuated_this_call = False
+ second = wu._actuate_output_contract_bail(
+ ctx,
+ blockers=["static_return_envelope_unavailable"],
+ target_code='await page.locator("#results").click()',
+ required_paths={"output.other_id"},
+ signature="sig-b",
+ current_fingerprint="fp-b",
+ )
+
+ assert ctx.latest_evaluate_result_composition_signature == "sig-a"
+ assert first.kind is not OutputContractActuationKind.BLOCKED_TERMINAL
+ assert second.kind is OutputContractActuationKind.STRUCTURE_DIRECTIVE
+ first_payload = events[0]["payload"]
+ second_payload = events[1]["payload"]
+ assert isinstance(first_payload, dict)
+ assert isinstance(second_payload, dict)
+ assert first_payload["loaded_result_source_producible"] is True
+ assert second_payload["loaded_result_source_producible"] is False
+
+
def _antecedent_ctx(*criteria: CompletionCriterion) -> CopilotContext:
ctx = make_copilot_ctx()
ctx.block_authoring_policy = wu.BlockAuthoringPolicy.CODE_ONLY_BROWSER
@@ -1736,30 +1826,6 @@ def test_runtime_repair_contract_carries_declaration_lane_with_stable_signature(
assert wu._stable_output_contract_key(wu._output_contract_scope_key(ctx), post_run.union) == pre_signature
-def test_declaration_only_production_still_counts_as_abstained_candidate() -> None:
- code = 'await page.click("#submit")\nreturn {"output": {"blocker": None}}'
-
- assert (
- wu._code_return_is_static_advisory_candidate(code, {"output.record_id", "output.blocker"}, {"output.blocker"})
- is True
- )
-
-
-def test_observation_production_disqualifies_advisory_candidate() -> None:
- code = 'value = await page.inner_text("#r")\nreturn {"output": {"record_id": value}}'
-
- assert (
- wu._code_return_is_static_advisory_candidate(code, {"output.record_id", "output.blocker"}, {"output.blocker"})
- is False
- )
-
-
-def test_declaration_only_contract_is_never_advisory_candidate() -> None:
- code = 'await page.click("#submit")'
-
- assert wu._code_return_is_static_advisory_candidate(code, {"output.blocker"}, {"output.blocker"}) is False
-
-
def _declaration_waiver_yaml(code_body: str) -> str:
indented = "\n".join(f" {line}" for line in code_body.splitlines())
return (
@@ -1785,7 +1851,7 @@ def _declaration_waiver_metadata(union: set[str]) -> list[dict[str, object]]:
]
-def test_post_steering_advisory_never_waives_declaration_return_miss() -> None:
+def test_typed_advisory_grant_never_waives_declaration_return_miss() -> None:
ctx = _antecedent_ctx(
CompletionCriterion(
id="c_record", outcome="The returned record includes record id.", output_path="output.record_id"
@@ -1794,7 +1860,7 @@ def test_post_steering_advisory_never_waives_declaration_return_miss() -> None:
)
union = {"output.record_id", "output.blocker"}
signature = wu._stable_output_contract_key(wu._output_contract_scope_key(ctx), union)
- ctx.output_contract_reject_count_by_signature = {signature: wu._MAX_OUTPUT_CONTRACT_STEERING_REJECTS}
+ wu._grant_output_contract_advisory_run(ctx, signature)
workflow_yaml = _declaration_waiver_yaml("result = await collect()\nreturn result")
evaluation = wu._evaluate_output_contract_for_code_block(
@@ -1802,7 +1868,7 @@ def test_post_steering_advisory_never_waives_declaration_return_miss() -> None:
)
assert evaluation is not None
- assert evaluation.payload["post_steering_static_return_advisory"] is True
+ assert evaluation.payload["actuated_static_return_advisory"] is True
assert evaluation.payload["static_return_advisory_paths"] == ["output.record_id"]
assert evaluation.missing_return_paths == ["output.blocker"]
assert evaluation.can_attempt_run is False
@@ -1817,7 +1883,7 @@ def test_stamped_declaration_with_advisory_accepts_run() -> None:
)
union = {"output.record_id", "output.blocker"}
signature = wu._stable_output_contract_key(wu._output_contract_scope_key(ctx), union)
- ctx.output_contract_reject_count_by_signature = {signature: wu._MAX_OUTPUT_CONTRACT_STEERING_REJECTS}
+ wu._grant_output_contract_advisory_run(ctx, signature)
workflow_yaml = _declaration_waiver_yaml('await page.click("#submit")\nreturn {"output": {"blocker": None}}')
evaluation = wu._evaluate_output_contract_for_code_block(
diff --git a/tests/unit/test_evaluate_actionable_target.py b/tests/unit/test_evaluate_actionable_target.py
index 328262ac2..1069b77d7 100644
--- a/tests/unit/test_evaluate_actionable_target.py
+++ b/tests/unit/test_evaluate_actionable_target.py
@@ -4,6 +4,7 @@ import json
import pytest
+from skyvern.forge.sdk.copilot.result_evidence import loaded_result_source_producible
from skyvern.forge.sdk.copilot.runtime import AgentContext
from skyvern.forge.sdk.copilot.tools import scouting
@@ -109,6 +110,68 @@ def _ctx() -> AgentContext:
return AgentContext.__new__(AgentContext)
+async def test_evaluate_mints_current_card_and_table_source_idempotently(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ packet = _packet()
+ packet["result_containers"] = [
+ {"selector": "#coastalCard", "text": "Provider credentialing status is visible"},
+ {"tag": "table", "selector": "#results", "row_count": 2, "sample_rows": ["A", "B"]},
+ ]
+ ctx = _ctx()
+ ctx.discovery_mcp_server = object()
+
+ async def fake_structured_evidence(*args: object, **kwargs: object) -> dict[str, object]:
+ return packet
+
+ monkeypatch.setattr(scouting, "_composition_get_structured_evidence", fake_structured_evidence)
+
+ page_evidence = await scouting._scout_act_observe_page_evidence(ctx, url="https://example.com/results")
+ assert getattr(ctx, "latest_evaluate_result_composition_steer", None) is None
+ result = {"ok": True, "data": {"url": "https://example.com/results"}}
+ await scouting._maybe_steer_evaluate_to_action(
+ ctx, result, url="https://example.com/results", page_evidence=page_evidence
+ )
+ first_carrier = ctx.latest_evaluate_result_composition_steer
+ await scouting._maybe_steer_evaluate_to_action(
+ ctx, result, url="https://example.com/results", page_evidence=page_evidence
+ )
+
+ assert first_carrier == ctx.latest_evaluate_result_composition_steer
+ assert [target.selector for target in first_carrier.targets] == ["#coastalCard", "#results"]
+ assert first_carrier.source_tool == "evaluate"
+ assert first_carrier.source_url == "https://example.com/results"
+ assert loaded_result_source_producible(first_carrier) is True
+ assert loaded_result_source_producible(first_carrier, target_code='await page.locator("#coastalCard")') is True
+ assert loaded_result_source_producible(first_carrier, target_code='await page.locator("#otherCard")') is False
+
+
+async def test_scout_interaction_observation_does_not_mint_evaluate_source(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ packet = _packet()
+ packet["result_containers"] = [{"selector": "#coastalCard", "text": "Provider result is visible"}]
+ ctx = _ctx()
+ ctx.discovery_mcp_server = object()
+
+ async def fake_structured_evidence(*args: object, **kwargs: object) -> dict[str, object]:
+ return packet
+
+ monkeypatch.setattr(scouting, "_composition_get_structured_evidence", fake_structured_evidence)
+
+ await scouting._register_scout_interaction_observation(
+ ctx,
+ tool_name="click",
+ selector="#search",
+ source_url="https://example.com/search",
+ url="https://example.com/results",
+ )
+
+ assert getattr(ctx, "latest_evaluate_result_composition_steer", None) is None
+
+
async def _seed(ctx, monkeypatch, packet, url):
async def fake(c, *, url):
return packet
@@ -209,6 +272,8 @@ async def test_first_loaded_result_table_steers_to_composition_not_click_even_wi
}
assert ctx.latest_evaluate_result_composition_steer.result_container_count == 1
assert ctx.latest_evaluate_result_composition_steer.targets[0].selector == "#results"
+ assert ctx.latest_evaluate_result_composition_steer.source_tool == "evaluate"
+ assert ctx.latest_evaluate_result_composition_steer.source_url == "https://example.com/results"
assert ctx.last_evaluate_actionable_signature is None
assert ctx.last_evaluate_actionable_url is None