Skip to content

Automate browser based workflows with AI

fix(SKY-11899): Copilot fresh-session replay frontier (#7149)

Andrew Neilsonf22243ea3fdd
diff --git a/skyvern/config.py b/skyvern/config.py
index 4c00e72e5..93f3506ea 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -202,11 +202,6 @@ class Settings(BaseSettings):
WORKFLOW_COPILOT_CODE_BLOCK_MODE: bool = False
# Default code_only for MCP block/workflow tools. Off = permissive.
MCP_CODE_ONLY_MODE: bool = False
- # Any copilot test-run whose leading block replays a login fill on a scout-authenticated
- # workflow runs in a fresh browser session, so that fill is not replayed into the scout's
- # already-authenticated session (the first run and every login-replaying repair re-run alike).
- # Off (default) reuses the scout debug session (SKY-9328) for every run as today.
- COPILOT_FRESH_SESSION_FIRST_SYNTHESIZED_TEST_RUN: bool = False
# Default for the bounded code-block self-heal; off by default.
ENABLE_CODE_BLOCK_SELF_HEALING: bool = False
PORT: int = 8000
diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.py
index a749b8b24..e28a1544f 100644
--- a/skyvern/forge/sdk/copilot/build_test_outcome.py
+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py
@@ -87,6 +87,8 @@ class RecordedBuildTestOutcome(BaseModel):
observed_evidence_summary: str = ""
workflow_run_id: str | None = None
block_labels: list[str] = Field(default_factory=list)
+ requested_block_labels: list[str] = Field(default_factory=list)
+ block_shape_hashes: dict[str, str] = Field(default_factory=dict)
structural_failure_identity: str = ""
verified_progress_marker: str = ""
page_evidence_refs: list[str] = Field(default_factory=list)
@@ -851,6 +853,8 @@ def _required_input_unbound_identity(
def _required_input_unbound_outcome(
failed_block: Mapping[str, object] | None,
block_labels: list[str],
+ requested_block_labels: list[str],
+ block_shape_hashes: Mapping[str, str],
workflow_run_id: str | None,
authored_structure_signature: str | None,
referenced_unbound_keys: Sequence[str],
@@ -863,6 +867,8 @@ def _required_input_unbound_outcome(
reason_code="required_input_unbound",
workflow_run_id=workflow_run_id or None,
block_labels=block_labels,
+ requested_block_labels=requested_block_labels,
+ block_shape_hashes=dict(block_shape_hashes),
structural_failure_identity=_required_input_unbound_identity(failed_block, referenced_unbound_keys),
authored_structure_signature=authored_structure_signature,
observed_evidence_summary=_bounded_text(
@@ -886,12 +892,18 @@ def recorded_outcome_from_run_blocks_result(
registered_output_parameter_payloads: Sequence[Mapping[str, object]] | None = None,
unbound_required_parameter_keys: Sequence[str] | None = None,
block_parameter_keys: Mapping[str, Sequence[str]] | None = None,
+ block_shape_hashes: Mapping[str, str] | None = None,
) -> RecordedBuildTestOutcome | None:
data = _dict(result.get("data"))
workflow_run_id = _safe_str(data.get("workflow_run_id"))
blocks = _block_dicts(data.get("blocks"))
failed_block = _first_failed_block(blocks)
block_labels = [_safe_str(block.get("label")) for block in blocks if _safe_str(block.get("label"))]
+ requested = data.get("requested_block_labels")
+ requested_block_labels = (
+ _clean_list([label for label in requested if isinstance(label, str)]) if isinstance(requested, list) else []
+ )
+ block_shape_hashes = dict(block_shape_hashes or {})
referenced_unbound_keys = _referenced_unbound_input_keys(
result,
failed_block,
@@ -921,6 +933,8 @@ def recorded_outcome_from_run_blocks_result(
reason_code=reason_code,
workflow_run_id=recorded_run_outcome.workflow_run_id or workflow_run_id or None,
block_labels=block_labels,
+ requested_block_labels=requested_block_labels,
+ block_shape_hashes=block_shape_hashes,
authored_structure_signature=authored_structure_signature,
observed_evidence_summary=recorded_run_outcome.display_reason or "",
key_provenance={"structural_failure_identity": "terminal blocker precedence suppresses repair prompt"},
@@ -933,6 +947,8 @@ def recorded_outcome_from_run_blocks_result(
reason_code="verified_success",
workflow_run_id=recorded_run_outcome.workflow_run_id or workflow_run_id or None,
block_labels=block_labels,
+ requested_block_labels=requested_block_labels,
+ block_shape_hashes=block_shape_hashes,
verified_progress_marker=verification_identity or "run_completed_verified",
evidence_refs=output_refs,
authored_structure_signature=authored_structure_signature,
@@ -950,6 +966,8 @@ def recorded_outcome_from_run_blocks_result(
reason_code=reason_code,
workflow_run_id=recorded_run_outcome.workflow_run_id or workflow_run_id or None,
block_labels=block_labels,
+ requested_block_labels=requested_block_labels,
+ block_shape_hashes=block_shape_hashes,
authored_structure_signature=authored_structure_signature,
observed_evidence_summary=recorded_run_outcome.display_reason or "",
key_provenance={"structural_failure_identity": "run outcome was not evaluated"},
@@ -958,6 +976,8 @@ def recorded_outcome_from_run_blocks_result(
return _required_input_unbound_outcome(
failed_block,
block_labels,
+ requested_block_labels,
+ block_shape_hashes,
recorded_run_outcome.workflow_run_id or workflow_run_id or None,
authored_structure_signature,
referenced_unbound_keys,
@@ -972,6 +992,8 @@ def recorded_outcome_from_run_blocks_result(
reason_code=reason_code,
workflow_run_id=recorded_run_outcome.workflow_run_id or workflow_run_id or None,
block_labels=block_labels,
+ requested_block_labels=requested_block_labels,
+ block_shape_hashes=block_shape_hashes,
authored_structure_signature=authored_structure_signature,
observed_evidence_summary=recorded_run_outcome.display_reason or "",
key_provenance={"structural_failure_identity": "no typed verification/page/output identity available"},
@@ -988,6 +1010,8 @@ def recorded_outcome_from_run_blocks_result(
reason_code="fallback_floor_turn_unsatisfiable",
workflow_run_id=recorded_run_outcome.workflow_run_id or workflow_run_id or None,
block_labels=block_labels,
+ requested_block_labels=requested_block_labels,
+ block_shape_hashes=block_shape_hashes,
authored_structure_signature=authored_structure_signature,
observed_evidence_summary=recorded_run_outcome.display_reason or "",
key_provenance={"structural_failure_identity": "turn-unsatisfiable fallback floor, no reachable route"},
@@ -999,6 +1023,8 @@ def recorded_outcome_from_run_blocks_result(
reason_code=reason_code,
workflow_run_id=recorded_run_outcome.workflow_run_id or workflow_run_id or None,
block_labels=block_labels,
+ requested_block_labels=requested_block_labels,
+ block_shape_hashes=block_shape_hashes,
structural_failure_identity=structural_identity,
page_evidence_refs=page_refs,
evidence_refs=evidence_refs,
@@ -1023,6 +1049,8 @@ def recorded_outcome_from_run_blocks_result(
return _required_input_unbound_outcome(
failed_block,
block_labels,
+ requested_block_labels,
+ block_shape_hashes,
workflow_run_id or None,
authored_structure_signature,
referenced_unbound_keys,
@@ -1062,6 +1090,8 @@ def recorded_outcome_from_run_blocks_result(
reason_code=reason_code,
workflow_run_id=workflow_run_id or None,
block_labels=block_labels,
+ requested_block_labels=requested_block_labels,
+ block_shape_hashes=block_shape_hashes,
structural_failure_identity=structural_identity,
page_evidence_refs=page_refs,
evidence_refs=output_refs,
diff --git a/skyvern/forge/sdk/copilot/tools/__init__.py b/skyvern/forge/sdk/copilot/tools/__init__.py
index c4320d2fe..8d0ce4ca4 100644
--- a/skyvern/forge/sdk/copilot/tools/__init__.py
+++ b/skyvern/forge/sdk/copilot/tools/__init__.py
@@ -702,6 +702,7 @@ async def update_and_run_blocks_tool(
"raw_code_artifact_metadata": serialized_code_artifact_metadata
if scaffold_applied or envelope_imposed
else code_artifact_metadata,
+ "block_labels": block_labels,
},
copilot_ctx,
allow_missing_credentials=skip_run_after_update,
diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.py
index 5a70c642e..443ae6e00 100644
--- a/skyvern/forge/sdk/copilot/tools/completion.py
+++ b/skyvern/forge/sdk/copilot/tools/completion.py
@@ -589,15 +589,23 @@ def _download_file_name(value: Any) -> str | None:
def _completion_evidence_payload(output: Any) -> Any:
- if not isinstance(output, dict) or not any(output.get(key) for key in REGISTERED_DOWNLOAD_OUTPUT_KEYS):
+ if not isinstance(output, dict):
return output
+ nested_output = output.get("output")
+ has_download_output = any(output.get(key) for key in REGISTERED_DOWNLOAD_OUTPUT_KEYS)
+ has_nested_download_output = isinstance(nested_output, dict) and any(
+ nested_output.get(key) for key in REGISTERED_DOWNLOAD_OUTPUT_KEYS
+ )
+ if not has_download_output and not has_nested_download_output:
+ return output
+ download_output = output if has_download_output or not isinstance(nested_output, dict) else nested_output
names: list[str] = []
- if name := _download_file_name(output.get("downloaded_file_name")):
+ if name := _download_file_name(download_output.get("downloaded_file_name")):
names.append(name)
- files = output.get("downloaded_files")
+ files = download_output.get("downloaded_files")
if isinstance(files, list):
names.extend(name for item in files[:_MAX_EVIDENCE_FILE_NAMES] if (name := _download_file_name(item)))
- urls = output.get("downloaded_file_urls")
+ urls = download_output.get("downloaded_file_urls")
if isinstance(urls, list):
names.extend(name for item in urls[:_MAX_EVIDENCE_FILE_NAMES] if (name := _download_file_name(item)))
payload: dict[str, Any] = {"download_registered": True}
@@ -605,11 +613,19 @@ def _completion_evidence_payload(output: Any) -> Any:
payload["downloaded_file_count"] = len(files)
if isinstance(urls, list):
payload["downloaded_file_url_count"] = len(urls)
- artifacts = output.get("downloaded_file_artifact_ids")
+ artifacts = download_output.get("downloaded_file_artifact_ids")
if isinstance(artifacts, list):
payload["downloaded_file_artifact_count"] = len(artifacts)
if names:
payload["downloaded_file_names"] = list(dict.fromkeys(names))[:_MAX_EVIDENCE_FILE_NAMES]
+ if has_nested_download_output and not has_download_output and isinstance(nested_output, dict):
+ preserved_output = {
+ key: value
+ for key, value in nested_output.items()
+ if key not in REGISTERED_DOWNLOAD_OUTPUT_KEYS and key not in {"page", "download"}
+ }
+ if preserved_output:
+ payload["output"] = preserved_output
return payload
diff --git a/skyvern/forge/sdk/copilot/tools/frontier.py b/skyvern/forge/sdk/copilot/tools/frontier.py
index 6d719c6a5..1411ca590 100644
--- a/skyvern/forge/sdk/copilot/tools/frontier.py
+++ b/skyvern/forge/sdk/copilot/tools/frontier.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+import hashlib
import json
import re
from collections import Counter
@@ -21,6 +22,7 @@ from skyvern.forge.sdk.copilot.block_goal_wrapping import wrap_workflow_block_go
from skyvern.forge.sdk.copilot.build_phase import (
BuildPhase,
)
+from skyvern.forge.sdk.copilot.build_test_outcome import RecordedBuildTestOutcome
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.failure_tracking import (
_canonical_block_config,
@@ -693,10 +695,11 @@ def _plan_frontier(
verified_outputs: dict[str, Any] = dict(ctx.verified_block_outputs or {})
verified_prefix: list[str] = list(ctx.verified_prefix_labels or [])
verified_prefix_set = set(verified_prefix)
+ failed_frontier_label = _recorded_failed_frontier_label(ctx, requested_labels, new_definition)
# No old definition (cold start or parse failure) OR no diff signal → plain path.
if old_definition is None:
- frontier = requested_labels[0]
+ frontier = failed_frontier_label or requested_labels[0]
return _seed_for_frontier(requested_labels, frontier, verified_outputs, new_definition)
try:
@@ -707,6 +710,13 @@ def _plan_frontier(
earliest = _earliest_invalidated(requested_labels, invalidated)
if earliest is None:
+ if failed_frontier_label is not None:
+ return _seed_for_frontier(
+ requested_labels,
+ failed_frontier_label,
+ verified_outputs,
+ new_definition,
+ )
# No invalidation at all — unchanged request. Continue from the
# first unverified requested label so a model may keep passing the
# complete chain while the tool advances the browser in small
@@ -731,6 +741,22 @@ def _plan_frontier(
return _seed_for_frontier(requested_labels, requested_labels[0], verified_outputs, new_definition)
+ earliest_idx = requested_labels.index(earliest)
+ failed_frontier_idx = (
+ requested_labels.index(failed_frontier_label) if failed_frontier_label in requested_labels else None
+ )
+ if (
+ failed_frontier_label is not None
+ and failed_frontier_idx is not None
+ and (failed_frontier_idx < earliest_idx or failed_frontier_label != _recorded_failed_attempted_label(ctx))
+ ):
+ return _seed_for_frontier(
+ requested_labels,
+ failed_frontier_label,
+ verified_outputs,
+ new_definition,
+ )
+
# Ensure the prefix before the earliest invalidated label is all in the
# verified prefix from a successful prior run. Otherwise we have no
# trusted anchor — fall back to the full requested list.
@@ -777,6 +803,60 @@ def _first_unverified_requested_label(requested_labels: list[str], verified_pref
return None
+def _frontier_label_shape_hashes(labels: list[str], workflow_definition: object | None) -> dict[str, str] | None:
+ by_label = _blocks_by_label(workflow_definition)
+ hashes: dict[str, str] = {}
+ for label in labels:
+ block = by_label.get(label)
+ if block is None:
+ return None
+ shape = _canonical_block_config(block)
+ shape.pop("label", None)
+ serialized = json.dumps(shape, sort_keys=True, default=str, separators=(",", ":"))
+ hashes[label] = hashlib.sha256(serialized.encode("utf-8")).hexdigest()
+ return hashes
+
+
+def _recorded_failed_attempted_label(ctx: AgentContext) -> str | None:
+ outcome = ctx.latest_recorded_build_test_outcome
+ if not isinstance(outcome, RecordedBuildTestOutcome) or not outcome.attempted_block_label:
+ return None
+ return outcome.attempted_block_label
+
+
+def _recorded_failed_frontier_label(
+ ctx: AgentContext,
+ requested_labels: list[str],
+ new_definition: object | None = None,
+) -> str | None:
+ outcome = ctx.latest_recorded_build_test_outcome
+ if not isinstance(outcome, RecordedBuildTestOutcome):
+ return None
+ if outcome.phase != "persisted_block_run" or outcome.verdict != "repairable_failure":
+ return None
+ if not outcome.attempted_block_label:
+ return None
+ recorded_requested_labels = outcome.requested_block_labels or outcome.block_labels
+ if len(recorded_requested_labels) != len(requested_labels):
+ return None
+ if recorded_requested_labels == requested_labels:
+ return outcome.attempted_block_label if outcome.attempted_block_label in requested_labels else None
+
+ recorded_shape_hashes = outcome.block_shape_hashes
+ attempted_shape = recorded_shape_hashes.get(outcome.attempted_block_label)
+ if not attempted_shape:
+ return None
+ current_shape_hashes = _frontier_label_shape_hashes(requested_labels, new_definition)
+ if current_shape_hashes is None:
+ return None
+ recorded_chain_shapes = [recorded_shape_hashes.get(label) for label in recorded_requested_labels]
+ current_chain_shapes = [current_shape_hashes.get(label) for label in requested_labels]
+ if any(shape is None for shape in recorded_chain_shapes) or recorded_chain_shapes != current_chain_shapes:
+ return None
+ candidates = [label for label in requested_labels if current_shape_hashes.get(label) == attempted_shape]
+ return candidates[0] if len(candidates) == 1 else None
+
+
def _seed_for_frontier(
requested_labels: list[str],
frontier: str,
@@ -962,6 +1042,7 @@ def _frontier_runtime_anchor_url(ctx: CopilotContext, block_outputs_to_seed: dic
for value in (
ctx.verified_prefix_current_url,
getattr(ctx, "composition_page_evidence", None),
+ _recorded_failed_run_anchor_url(ctx),
):
candidate = _frontier_anchor_url_from_value(value)
if candidate is not None:
@@ -1001,7 +1082,36 @@ def _has_verified_prefix_before_frontier(
if not prefix_labels:
return False
verified = set(ctx.verified_prefix_labels or [])
- return all(label in verified for label in prefix_labels)
+ return all(label in verified for label in prefix_labels) or _has_recorded_failed_prefix_before_frontier(
+ ctx,
+ workflow_definition,
+ frontier_label,
+ )
+
+
+def _has_recorded_failed_prefix_before_frontier(
+ ctx: CopilotContext, workflow_definition: object | None, frontier_label: str | None
+) -> bool:
+ if not frontier_label:
+ return False
+ workflow_labels = _workflow_definition_block_labels(workflow_definition)
+ try:
+ frontier_idx = workflow_labels.index(frontier_label)
+ except ValueError:
+ return False
+ if frontier_idx == 0:
+ return False
+ return _recorded_failed_frontier_label(ctx, workflow_labels, workflow_definition) == frontier_label
+
+
+def _recorded_failed_run_anchor_url(ctx: CopilotContext) -> str | None:
+ outcome = ctx.latest_recorded_build_test_outcome
+ if not isinstance(outcome, RecordedBuildTestOutcome) or not outcome.workflow_run_id:
+ return None
+ evidence = ctx.workflow_verification_evidence
+ if evidence.workflow_run_id != outcome.workflow_run_id:
+ return None
+ return evidence.current_url
def _workflow_with_runtime_frontier_anchor(
@@ -1063,8 +1173,10 @@ async def _workflow_with_runtime_frontier_starter_url_seed(
*,
labels_to_execute: list[str],
runtime_frontier_anchor_url: str | None,
+ session_id_override: str | None = None,
) -> Workflow:
- if not labels_to_execute or runtime_frontier_anchor_url is None or not ctx.browser_session_id:
+ session_id = session_id_override or ctx.browser_session_id
+ if not labels_to_execute or runtime_frontier_anchor_url is None or not session_id:
return workflow
first_label = labels_to_execute[0]
@@ -1080,7 +1192,7 @@ async def _workflow_with_runtime_frontier_starter_url_seed(
current_page_url: str | None = None
try:
browser_state = await app.PERSISTENT_SESSIONS_MANAGER.get_browser_state(
- session_id=ctx.browser_session_id,
+ session_id=session_id,
organization_id=ctx.organization_id,
)
if browser_state is not None:
@@ -1090,7 +1202,7 @@ async def _workflow_with_runtime_frontier_starter_url_seed(
except Exception:
LOG.debug(
"Failed to inspect runtime frontier browser page before starter URL seed",
- browser_session_id=ctx.browser_session_id,
+ browser_session_id=session_id,
frontier_start_label=first_label,
exc_info=True,
)
@@ -1098,7 +1210,7 @@ async def _workflow_with_runtime_frontier_starter_url_seed(
if not _blank_runtime_page_url(current_page_url):
LOG.info(
"Preserved attached runtime frontier browser page",
- browser_session_id=ctx.browser_session_id,
+ browser_session_id=session_id,
frontier_start_label=first_label,
current_url=current_page_url,
continuation_url=runtime_frontier_anchor_url,
@@ -1113,7 +1225,7 @@ async def _workflow_with_runtime_frontier_starter_url_seed(
seeded_block.url = runtime_frontier_anchor_url
LOG.info(
"Seeded runtime frontier starter URL because attached browser page was blank",
- browser_session_id=ctx.browser_session_id,
+ browser_session_id=session_id,
frontier_start_label=first_label,
current_url=current_page_url,
continuation_url=runtime_frontier_anchor_url,
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index 3820b6c81..747f2d1be 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -178,6 +178,7 @@ from .credentials import (
from .frontier import (
_MAX_INCREMENTAL_PAGE_FRONTIER_LABELS,
_blocks_by_label,
+ _frontier_label_shape_hashes,
_workflow_with_runtime_block_goal_context,
_workflow_with_runtime_frontier_anchor,
_workflow_with_runtime_frontier_starter_url_seed,
@@ -813,22 +814,22 @@ def _should_use_fresh_session_for_login_first_replay(
"""Fresh session when this run replays a login fill into the scout's authenticated session.
Keyed on two planes the agent cannot edit between runs — the scout trajectory authenticated
- via a credential fill and this run's leading block fills one; frontier re-runs seeded past
- login carry no leading credential fill and keep reusing the scout session.
+ via a credential fill and any executed block in this run fills one; frontier re-runs seeded
+ past login carry no credential fill and keep reusing the scout session.
"""
- if not SettingsManager.get_settings().COPILOT_FRESH_SESSION_FIRST_SYNTHESIZED_TEST_RUN:
- return False
if not trajectory_has_credential_fill(ctx.scout_trajectory):
return False
return _labels_replay_login_fill(labels_to_execute, workflow)
def _labels_replay_login_fill(labels_to_execute: list[str], workflow: Workflow | None) -> bool:
- if not labels_to_execute:
+ if not labels_to_execute or workflow is None:
return False
- by_label = _blocks_by_label(workflow.workflow_definition if workflow else None)
- code = getattr(by_label.get(labels_to_execute[0]), "code", None)
- return isinstance(code, str) and code_contains_credential_fill(code)
+ code_inputs = _selected_code_security_inputs(
+ _workflow_definition_blocks_for_code_security(workflow.workflow_definition),
+ selected_labels=set(labels_to_execute),
+ )
+ return any(code_contains_credential_fill(code_input.code) for code_input in code_inputs)
def _runtime_code_security_failure_for_selected_labels(
@@ -1525,6 +1526,13 @@ async def _run_blocks_and_collect_debug(
run_session_id = ctx.browser_session_id
ctx.browser_session_id = debug_session_id
used_fresh_run_session = True
+ LOG.info(
+ "copilot_login_replay_fresh_session_minted",
+ labels_to_execute=labels_to_execute,
+ frontier_start_label=frontier_start_label,
+ run_session_id=run_session_id,
+ debug_session_id=debug_session_id,
+ )
else:
session_err = await ensure_browser_session(ctx)
if session_err is not None:
@@ -1536,6 +1544,7 @@ async def _run_blocks_and_collect_debug(
ctx,
labels_to_execute=labels_to_execute,
runtime_frontier_anchor_url=runtime_frontier_anchor_url,
+ session_id_override=run_session_id,
)
runtime_frontier_starter_url_seeded = seeded_runtime_workflow is not runtime_workflow
runtime_workflow = seeded_runtime_workflow
@@ -2832,6 +2841,19 @@ def _record_adjudicated_build_test_outcome(
)
workflow_yaml = copilot_ctx.workflow_yaml
code_artifact_metadata = copilot_ctx.code_artifact_metadata
+ workflow_definition = getattr(copilot_ctx.last_workflow, "workflow_definition", None)
+ raw_requested_block_labels = getattr(copilot_ctx, "last_requested_block_labels", None)
+ result_requested_block_labels = result_data.get("requested_block_labels") if isinstance(result_data, dict) else None
+ raw_requested_values: list[Any] = (
+ raw_requested_block_labels
+ if isinstance(raw_requested_block_labels, list)
+ else (result_requested_block_labels if isinstance(result_requested_block_labels, list) else [])
+ )
+ requested_block_labels = [label for label in raw_requested_values if isinstance(label, str)]
+ block_shape_hashes = _frontier_label_shape_hashes(
+ requested_block_labels,
+ workflow_definition,
+ )
record_build_test_outcome(
copilot_ctx,
recorded_outcome_from_run_blocks_result(
@@ -2849,6 +2871,7 @@ def _record_adjudicated_build_test_outcome(
workflow_yaml,
code_artifact_metadata,
),
+ block_shape_hashes=block_shape_hashes or {},
),
)
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index 8ff56d309..0eef5d301 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -1113,7 +1113,18 @@ def _adopt_exact_declared_parameter_keys_for_unresolved_names(workflow_yaml: str
if not isinstance(parsed, dict):
return workflow_yaml
declared_string_keys = _declared_string_workflow_parameter_keys(parsed)
- if not declared_string_keys:
+ workflow_definition = parsed.get("workflow_definition")
+ parameters = workflow_definition.get("parameters") if isinstance(workflow_definition, dict) else None
+ declared_credential_keys: set[str] = set()
+ if isinstance(parameters, list):
+ declared_credential_keys = {
+ str(parameter.get("key") or "").strip()
+ for parameter in parameters
+ if isinstance(parameter, dict)
+ if str(parameter.get("key") or "").strip() and _is_credential_parameter(parameter)
+ }
+ declared_adoptable_keys = declared_string_keys | declared_credential_keys
+ if not declared_adoptable_keys:
return workflow_yaml
available_binding_keys_by_label = _code_block_available_binding_keys_by_label(workflow_yaml)
adopted_by_label: dict[str, list[str]] = {}
@@ -1125,7 +1136,9 @@ def _adopt_exact_declared_parameter_keys_for_unresolved_names(workflow_yaml: str
if not code.strip():
continue
parameter_keys = _code_block_parameter_keys(block)
- available_declared_keys = set(available_binding_keys_by_label.get(label, [])) & declared_string_keys
+ available_declared_keys = (
+ set(available_binding_keys_by_label.get(label, [])) & declared_string_keys
+ ) | declared_credential_keys
if not available_declared_keys:
continue
diagnostic = sandbox_unresolved_name_repair_diagnostic(code, parameter_keys=parameter_keys)
@@ -5023,14 +5036,32 @@ def _author_time_reject_reopens_synthesized_imposition(ctx: AgentContext) -> boo
)
+def _recorded_outcome_imposition_block_labels(ctx: AgentContext) -> frozenset[str]:
+ constraint = ctx.recorded_outcome_binding_constraint
+ if isinstance(constraint, RecordedOutcomeBindingConstraint):
+ return frozenset(label.strip() for label in constraint.owning_block_labels if label.strip())
+ latest = ctx.latest_recorded_build_test_outcome
+ if isinstance(latest, RecordedBuildTestOutcome) and latest.is_authoritative:
+ return frozenset(label.strip() for label in latest.block_labels if label.strip())
+ return frozenset()
+
+
def _select_synthesized_imposition_code_block(
code_blocks: list[dict[str, Any]],
*,
prior_yaml: str | None,
+ preferred_labels: frozenset[str] = frozenset(),
) -> dict[str, Any] | None:
if len(code_blocks) == 1:
return code_blocks[0]
+ if preferred_labels:
+ preferred_matches = [
+ block for block in code_blocks if str(block.get("label") or "").strip() in preferred_labels
+ ]
+ if len(preferred_matches) == 1:
+ return preferred_matches[0]
+
synthesized_label_matches = [
block for block in code_blocks if str(block.get("label") or "").strip() == _SYNTHESIZED_BLOCK_LABEL
]
@@ -5384,6 +5415,7 @@ def _whole_trajectory_browser_surface_violations(
selected_code_block: dict[str, Any],
submitted_selected_code: str,
synthesized_code: str,
+ prior_yaml: str | None = None,
) -> _BrowserSurfaceValidation:
violations: list[str] = []
scouted_mutations, _, _ = _browser_surface_for_code(synthesized_code)
@@ -5391,6 +5423,8 @@ def _whole_trajectory_browser_surface_violations(
for block in code_blocks:
if block is selected_code_block:
continue
+ if prior_yaml is not None and not _submitted_code_block_changed(block, prior_yaml):
+ continue
label = _code_block_label(block)
block_code = str(block.get("code") or "")
block_mutations, _, block_ambiguous = _browser_surface_for_code(block_code)
@@ -6674,7 +6708,11 @@ def _maybe_impose_synthesized_code_block(workflow_yaml: str, ctx: AgentContext)
return _SynthesizedCodeImpositionResult(workflow_yaml=workflow_yaml)
code_blocks = _workflow_code_blocks(parsed)
prior_source, prior_yaml = _prior_yaml_source(ctx)
- code_block = _select_synthesized_imposition_code_block(code_blocks, prior_yaml=prior_yaml)
+ code_block = _select_synthesized_imposition_code_block(
+ code_blocks,
+ prior_yaml=prior_yaml,
+ preferred_labels=_recorded_outcome_imposition_block_labels(ctx),
+ )
if code_block is None:
return _SynthesizedCodeImpositionResult(workflow_yaml=workflow_yaml)
submitted_code = str(code_block.get("code") or "")
@@ -6734,6 +6772,7 @@ def _maybe_impose_synthesized_code_block(workflow_yaml: str, ctx: AgentContext)
selected_code_block=code_block,
submitted_selected_code=submitted_code,
synthesized_code=synthesized.code,
+ prior_yaml=prior_yaml,
)
violations.extend(surface_validation.violations)
extraction_suffix = _submitted_suffix_after_synthesized_code(submitted_code, synthesized.code)
@@ -7783,6 +7822,8 @@ def _artifact_string_list(value: Any) -> list[str]:
def _credentialed_code_block_scout_gate_errors(
workflow_yaml: str | None,
ctx: AgentContext,
+ *,
+ block_labels: Iterable[str] | None = None,
) -> list[str]:
if _copilot_block_authoring_policy(ctx) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
return []
@@ -7797,9 +7838,25 @@ def _credentialed_code_block_scout_gate_errors(
credential_params_by_key = credential_param_ids(workflow_definition.get("parameters"))
if not credential_params_by_key:
return []
+ prior_blocks_by_label = {}
+ prior_credential_params_by_key = {}
+ prior_workflow_yaml = ctx.workflow_yaml
+ if isinstance(prior_workflow_yaml, str) and prior_workflow_yaml.strip():
+ prior_parsed = parse_workflow_yaml(prior_workflow_yaml)
+ if isinstance(prior_parsed, dict):
+ prior_workflow_definition = prior_parsed.get("workflow_definition")
+ if isinstance(prior_workflow_definition, dict):
+ prior_credential_params_by_key = credential_param_ids(prior_workflow_definition.get("parameters"))
+ for prior_block in workflow_blocks(prior_parsed):
+ if _enum_or_string_name(prior_block.get("block_type")) != BlockType.CODE.value:
+ continue
+ prior_label = str(prior_block.get("label") or "").strip()
+ if prior_label:
+ prior_blocks_by_label[prior_label] = prior_block
scout_trajectory = getattr(ctx, "scout_trajectory", None)
if not isinstance(scout_trajectory, list):
scout_trajectory = []
+ selected_block_labels = {label.strip() for label in block_labels or [] if label.strip()}
errors: list[str] = []
for block in workflow_blocks(parsed):
@@ -7808,6 +7865,23 @@ def _credentialed_code_block_scout_gate_errors(
code = str(block.get("code") or "")
if not code.strip():
continue
+ block_label = str(block.get("label") or "").strip()
+ if selected_block_labels and block_label not in selected_block_labels:
+ continue
+ matching_prior_block = prior_blocks_by_label.get(block_label) if block_label else None
+ if (
+ isinstance(matching_prior_block, dict)
+ and str(matching_prior_block.get("code") or "") == code
+ and _code_block_parameter_keys(matching_prior_block) == _code_block_parameter_keys(block)
+ ):
+ accessed_parameter_keys = {
+ access.parameter_key for access in _credential_field_accesses(code) if access.requires_live_scout
+ }
+ if accessed_parameter_keys and all(
+ prior_credential_params_by_key.get(parameter_key) == credential_params_by_key.get(parameter_key)
+ for parameter_key in accessed_parameter_keys
+ ):
+ continue
required_fields_by_parameter: dict[str, tuple[set[str], set[str]]] = {}
for access in _credential_field_accesses(code):
if not access.requires_live_scout:
@@ -7864,7 +7938,7 @@ def _credentialed_code_block_scout_gate_errors(
if not missing_fields and not missing_submit:
continue
- block_label = str(block.get("label") or "").strip() or "this code block"
+ block_label = block_label or "this code block"
requirements: list[str] = []
if missing_fields:
joined_fields = ", ".join(f"`{field}`" for field in missing_fields)
@@ -8184,7 +8258,7 @@ async def _update_workflow(
credential_scout_errors = (
[]
if _request_policy_allows_untested_code_block_draft(ctx)
- else _credentialed_code_block_scout_gate_errors(workflow_yaml, ctx)
+ else _credentialed_code_block_scout_gate_errors(workflow_yaml, ctx, block_labels=params.get("block_labels"))
)
unresolved_symbol_priority_reject = _is_unresolved_symbol_repair_context(code_authoring_repair_context)
credential_priority_reject = (
diff --git a/tests/unit/test_copilot_build_test_outcome.py b/tests/unit/test_copilot_build_test_outcome.py
index 7535e8268..18aa86d7f 100644
--- a/tests/unit/test_copilot_build_test_outcome.py
+++ b/tests/unit/test_copilot_build_test_outcome.py
@@ -49,6 +49,28 @@ def test_structural_key_changes_when_page_or_result_structure_changes() -> None:
assert first.structural_key != second.structural_key
+def test_run_blocks_outcome_records_requested_labels_and_shape_hashes() -> None:
+ outcome = recorded_outcome_from_run_blocks_result(
+ {
+ "ok": False,
+ "data": {
+ "workflow_run_id": "wr_failed",
+ "requested_block_labels": ["open", "search", "extract"],
+ "blocks": [
+ {"label": "search", "status": "failed", "failure_type": "runtime_error"},
+ ],
+ "overall_status": "failed",
+ "failure_type": "runtime_error",
+ },
+ },
+ block_shape_hashes={"open": "h1", "search": "h2", "extract": "h3"},
+ )
+
+ assert outcome is not None
+ assert outcome.requested_block_labels == ["open", "search", "extract"]
+ assert outcome.block_shape_hashes == {"open": "h1", "search": "h2", "extract": "h3"}
+
+
def test_structural_key_ignores_display_prose_and_workflow_run_id() -> None:
first = RecordedBuildTestOutcome(
phase="persisted_block_run",
diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.py
index 4f92ffb97..4d0d3a0c9 100644
--- a/tests/unit/test_copilot_code_block_persist_seam.py
+++ b/tests/unit/test_copilot_code_block_persist_seam.py
@@ -23,6 +23,7 @@ from skyvern.forge.sdk.copilot.blocker_signal import (
from skyvern.forge.sdk.copilot.build_test_outcome import (
BuildTestOutcomeReasonCode,
RecordedBuildTestOutcome,
+ RecordedOutcomeBindingConstraint,
authored_structure_signature_from_workflow,
latest_recorded_build_test_outcome_repeated,
recorded_outcome_from_author_time_reject,
@@ -6362,6 +6363,30 @@ class TestOutputContractBudgetRunGuard:
class TestCodeBlockParameterPersistSeam:
+ def test_declared_credential_key_is_adopted_for_unresolved_name(self) -> None:
+ workflow_yaml = _yaml(
+ """
+ title: Portal login
+ workflow_definition:
+ parameters:
+ - parameter_type: workflow
+ workflow_parameter_type: credential_id
+ key: portal_credentials
+ default_value: cred_123
+ blocks:
+ - block_type: code
+ label: sign_in
+ code: |
+ await page.locator("#user").fill(portal_credentials.username)
+ """
+ )
+
+ adopted = workflow_update_module._adopt_exact_declared_parameter_keys_for_unresolved_names(workflow_yaml)
+
+ parsed = parse_workflow_yaml(adopted)
+ blocks = workflow_blocks(parsed)
+ assert blocks[0]["parameter_keys"] == ["portal_credentials"]
+
@pytest.mark.asyncio
async def test_undeclared_parameter_key_rejects_before_persist(self, monkeypatch: pytest.MonkeyPatch) -> None:
_stub_successful_update(monkeypatch)
@@ -7212,6 +7237,92 @@ class TestCompiledAuthoringImposition:
assert blocks["summarize_statement"]["code"].strip() == 'return {"status": "ready"}'
assert "async with page.expect_download()" in blocks["scout_synthesized_browser_steps"]["code"]
+ def test_imposition_targets_recorded_outcome_owner_in_multi_code_workflow(self) -> None:
+ ctx = self._download_ctx()
+ ctx.recorded_outcome_binding_constraint = RecordedOutcomeBindingConstraint(
+ repeated_structural_key="recorded-download",
+ phase="persisted_block_run",
+ reason_code="outcome_not_demonstrated",
+ frontier_facet="value_shape",
+ owning_block_labels=["download_matching_invoice_pdf"],
+ )
+ workflow_yaml = _yaml(
+ """
+ title: Statement download
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: login_to_apex_business
+ code: |
+ return {"logged_in": True}
+ - block_type: code
+ label: download_matching_invoice_pdf
+ code: |
+ async with page.expect_download() as download_info:
+ await page.locator("a[href='/billing/statement.pdf']").click()
+ download = await download_info.value
+ return {"downloaded_files": [download.suggested_filename]}
+ """
+ )
+
+ result = workflow_update_module._maybe_impose_synthesized_code_block(workflow_yaml, ctx)
+
+ assert result.violations == []
+ assert result.substitutions is not None
+ parsed = parse_workflow_yaml(result.workflow_yaml)
+ assert isinstance(parsed, dict)
+ blocks = {str(block.get("label")): block for block in workflow_blocks(parsed)}
+ assert blocks["login_to_apex_business"]["code"].strip() == 'return {"logged_in": True}'
+ assert "async with page.expect_download()" in blocks["download_matching_invoice_pdf"]["code"]
+
+ def test_imposition_ignores_unchanged_persisted_browser_sibling(self) -> None:
+ ctx = self._download_ctx()
+ prior_yaml = _yaml(
+ """
+ title: Statement download
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: login_to_apex_business
+ code: |
+ await page.locator("#email").fill(str(login_credentials["username"]))
+ await page.locator("#password").fill(str(login_credentials["password"]))
+ await page.locator("#sign-in").click()
+ - block_type: code
+ label: download_matching_invoice_pdf
+ code: |
+ return {"status": "ready"}
+ """
+ )
+ submitted_yaml = _yaml(
+ """
+ title: Statement download
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: login_to_apex_business
+ code: |
+ await page.locator("#email").fill(str(login_credentials["username"]))
+ await page.locator("#password").fill(str(login_credentials["password"]))
+ await page.locator("#sign-in").click()
+ - block_type: code
+ label: download_matching_invoice_pdf
+ code: |
+ await page.locator("#statement-row").click()
+ """
+ )
+ ctx.workflow_yaml = prior_yaml
+
+ result = workflow_update_module._maybe_impose_synthesized_code_block(submitted_yaml, ctx)
+
+ assert result.violations == []
+ assert result.substitutions is not None
+ parsed = parse_workflow_yaml(result.workflow_yaml)
+ assert isinstance(parsed, dict)
+ blocks = {str(block.get("label")): block for block in workflow_blocks(parsed)}
+ assert "#sign-in" in str(blocks["login_to_apex_business"]["code"])
+ assert "async with page.expect_download()" in str(blocks["download_matching_invoice_pdf"]["code"])
+
def test_prior_synthesized_label_prevents_unrelated_multi_code_imposition(self) -> None:
ctx = self._download_ctx()
prior_yaml = _yaml(
@@ -8673,6 +8784,35 @@ class TestStaleLabelSeamFlow:
class TestCredentialScoutPersistGate:
+ _MULTI_BLOCK_TARGETED_CREDENTIAL_YAML = _yaml(
+ """
+ title: Saved credential login
+ workflow_definition:
+ parameters:
+ - parameter_type: workflow
+ workflow_parameter_type: credential_id
+ key: login_credential
+ default_value: cred_missing
+ blocks:
+ - block_type: code
+ label: enter_username
+ parameter_keys: [login_credential]
+ code: |
+ await page.locator("#email").fill(login_credential.username)
+ await page.locator("#continue").click()
+ - block_type: code
+ label: sign_in_to_business_center
+ parameter_keys: [login_credential]
+ code: |
+ await page.locator("input[type='password']").fill(login_credential.password)
+ await page.locator("#sign-in").click()
+ - block_type: code
+ label: open_matching_statement
+ code: |
+ text = await page.locator("table").inner_text()
+ print(text)
+ """
+ )
_SUBMIT_CODE_YAML = _credential_code_yaml(
code="""
await page.locator("#email").fill(login_credential.username)
@@ -8873,6 +9013,137 @@ class TestCredentialScoutPersistGate:
assert "fill_credential_field" not in error_text
assert "saved-credential login flow" not in error_text
+ @pytest.mark.asyncio
+ async def test_targeted_run_labels_scope_credential_scout_gate(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = []
+
+ accepted = await _update_workflow(
+ {
+ "workflow_yaml": self._MULTI_BLOCK_TARGETED_CREDENTIAL_YAML,
+ "block_labels": ["open_matching_statement"],
+ },
+ ctx,
+ allow_missing_credentials=True,
+ )
+
+ assert accepted["ok"] is True
+
+ selected_credential_ctx = _code_only_ctx()
+ selected_credential_ctx.scout_trajectory = []
+ rejected = await _update_workflow(
+ {
+ "workflow_yaml": self._MULTI_BLOCK_TARGETED_CREDENTIAL_YAML,
+ "block_labels": ["sign_in_to_business_center"],
+ },
+ selected_credential_ctx,
+ )
+
+ assert rejected["ok"] is False
+ assert rejected["data"]["failure_type"] == "missing_credential_or_init"
+ assert "sign_in_to_business_center" in rejected["error"]
+ assert rejected["user_facing_summary"] == CREDENTIAL_SCOUT_VERIFY_REPLY
+
+ @pytest.mark.asyncio
+ async def test_persisted_parameter_shape_does_not_rescout_unchanged_selected_credential_block(self) -> None:
+ sign_in_code = """
+ await page.locator("#email").fill(login_credential.username)
+ await page.locator("input[type='password']").fill(login_credential.password)
+ await page.locator("#sign-in").click()
+ """
+ sign_in_code = textwrap.dedent(sign_in_code).strip()
+ login_parameter = {
+ "parameter_type": "workflow",
+ "workflow_parameter_type": "credential_id",
+ "key": "login_credential",
+ "default_value": "cred_missing",
+ }
+ prior_yaml = yaml.safe_dump(
+ {
+ "title": "Saved credential login",
+ "workflow_definition": {
+ "parameters": [login_parameter],
+ "blocks": [
+ {
+ "block_type": "code",
+ "label": "sign_in_to_business_center",
+ "parameters": [login_parameter],
+ "code": sign_in_code,
+ }
+ ],
+ },
+ },
+ sort_keys=False,
+ )
+ submitted_yaml = yaml.safe_dump(
+ {
+ "title": "Saved credential login",
+ "workflow_definition": {
+ "parameters": [
+ login_parameter,
+ {
+ "parameter_type": "workflow",
+ "workflow_parameter_type": "string",
+ "key": "account_number",
+ "default_value": "100245",
+ },
+ ],
+ "blocks": [
+ {
+ "block_type": "code",
+ "label": "sign_in_to_business_center",
+ "parameter_keys": ["login_credential"],
+ "code": sign_in_code,
+ },
+ {
+ "block_type": "code",
+ "label": "open_matching_statement",
+ "parameter_keys": ["account_number"],
+ "code": 'await page.get_by_text("View Printable Statement").wait_for(timeout=5000)',
+ },
+ ],
+ },
+ },
+ sort_keys=False,
+ )
+ ctx = _code_only_ctx()
+ ctx.workflow_yaml = prior_yaml
+ ctx.scout_trajectory = []
+
+ result = await _update_workflow(
+ {
+ "workflow_yaml": submitted_yaml,
+ "block_labels": ["sign_in_to_business_center", "open_matching_statement"],
+ },
+ ctx,
+ )
+
+ assert result["ok"] is False
+ error_text = str(result.get("error") or "")
+ assert "open_matching_statement" in error_text
+ assert "fill_credential_field" not in error_text
+
+ changed_yaml = submitted_yaml.replace(
+ 'await page.locator("#sign-in").click()',
+ 'await page.locator("#sign-in").click()\n await page.locator("#post-login").click()',
+ )
+ changed_ctx = _code_only_ctx()
+ changed_ctx.workflow_yaml = prior_yaml
+ changed_ctx.scout_trajectory = []
+
+ changed_result = await _update_workflow(
+ {
+ "workflow_yaml": changed_yaml,
+ "block_labels": ["sign_in_to_business_center"],
+ },
+ changed_ctx,
+ )
+
+ assert changed_result["ok"] is False
+ assert changed_result["data"]["failure_type"] == "missing_credential_or_init"
+ assert changed_result["user_facing_summary"] == CREDENTIAL_SCOUT_VERIFY_REPLY
+
def test_run_id_leak_check_covers_non_numeric_ids() -> None:
with pytest.raises(ValueError):
@@ -9354,6 +9625,34 @@ class TestCodeAuthoringGuardrailChurnBackstop:
assert ctx.blocker_signal is None
assert ctx.latest_tool_blocker_signal is None
+ @pytest.mark.asyncio
+ async def test_unchanged_persisted_credential_block_does_not_require_new_scout(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ prior_yaml = _safe_credential_collision_yaml(0)
+ ctx = _code_only_ctx()
+ ctx.workflow_yaml = prior_yaml
+ ctx.scout_trajectory = []
+
+ result = await _update_workflow({"workflow_yaml": prior_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is True
+ assert result.get("user_facing_summary") != CREDENTIAL_SCOUT_VERIFY_REPLY
+ assert ctx.code_authoring_guardrail_reject_count == 0
+
+ @pytest.mark.asyncio
+ async def test_changed_persisted_credential_block_still_requires_new_scout(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.workflow_yaml = _safe_credential_collision_yaml(0)
+ ctx.scout_trajectory = []
+
+ result = await _update_workflow({"workflow_yaml": _safe_credential_collision_yaml(1)}, ctx)
+
+ assert result["ok"] is False
+ assert result["data"]["failure_type"] == "missing_credential_or_init"
+ assert result["user_facing_summary"] == CREDENTIAL_SCOUT_VERIFY_REPLY
+
@pytest.mark.asyncio
async def test_pure_credential_priority_churn_climbs_and_halts(self) -> None:
ctx = _code_only_ctx()
diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.py
index 8ed8a5c20..9b0d75368 100644
--- a/tests/unit/test_copilot_completion_verification.py
+++ b/tests/unit/test_copilot_completion_verification.py
@@ -8492,6 +8492,69 @@ async def test_download_registered_output_parameter_injects_and_verifies_without
assert ctx.request_policy.completion_criteria == []
+async def test_download_registered_nested_output_corroborates_requested_fields(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def fail_handler(**_: str) -> str:
+ raise AssertionError("nested registered download evidence must bypass the judge")
+
+ _patch_completion_handler(monkeypatch, fail_handler)
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "extract_profile")
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("account_number")
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ _criterion(
+ "c_account_number",
+ "The returned record includes account number.",
+ output_path="output.account_number",
+ ),
+ registered_download_completion_criterion(),
+ ]
+ )
+
+ run_result = _download_result(
+ {
+ "output": {
+ "account_number": "100245",
+ "downloaded_files": [{"filename": "statement.pdf"}],
+ "downloaded_file_urls": [],
+ "downloaded_file_artifact_ids": [],
+ }
+ }
+ )
+ run_result["data"]["executed_block_labels"] = ["extract_profile"]
+ run_result["data"]["blocks"] = [
+ {
+ "label": "extract_profile",
+ "block_type": "CODE",
+ "status": "completed",
+ "extracted_data": {
+ "output": {
+ "account_number": "100245",
+ "downloaded_files": [{"filename": "statement.pdf"}],
+ "downloaded_file_urls": [],
+ "downloaded_file_artifact_ids": [],
+ }
+ },
+ }
+ ]
+
+ verification = await _maybe_run_completion_verification(
+ ctx,
+ run_result,
+ time.monotonic(),
+ )
+
+ assert verification is not None
+ assert verification.is_fully_satisfied() is True
+ verdicts = {verdict.criterion_id: verdict for verdict in verification.verdicts}
+ assert verdicts["c_account_number"].reason_code == "structurally_abstained"
+ assert verdicts["c_account_number"].evidence_ref == "block_outputs:extract_profile.output.account_number"
+ assert verdicts[REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID].reason_code == "evidence_confirms"
+
+
@pytest.mark.asyncio
async def test_marked_requested_output_id_without_expected_value_is_no_evidence(
monkeypatch: pytest.MonkeyPatch,
diff --git a/tests/unit/test_copilot_frontier.py b/tests/unit/test_copilot_frontier.py
index 0c59fda43..3050cdb2b 100644
--- a/tests/unit/test_copilot_frontier.py
+++ b/tests/unit/test_copilot_frontier.py
@@ -9,6 +9,7 @@ import pytest
from jinja2.sandbox import SandboxedEnvironment
from skyvern.forge.sdk.copilot import tools
+from skyvern.forge.sdk.copilot.build_test_outcome import RecordedBuildTestOutcome
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.enforcement import (
MAX_FAILED_TEST_NUDGES,
@@ -118,6 +119,14 @@ class _FakeFailingPersistentSessionsManager:
raise RuntimeError("browser state unavailable")
+class _SessionKeyedPersistentSessionsManager:
+ def __init__(self, browser_state_by_session: dict[str, _FakeBrowserState]) -> None:
+ self._browser_state_by_session = browser_state_by_session
+
+ async def get_browser_state(self, session_id: str, organization_id: str) -> _FakeBrowserState | None:
+ return self._browser_state_by_session.get(session_id)
+
+
def _make_ctx(**kwargs: object) -> CopilotContext:
defaults: dict[str, Any] = dict(
organization_id="org",
@@ -363,6 +372,43 @@ async def test_runtime_frontier_starter_url_seed_fills_when_browser_state_lookup
assert workflow.workflow_definition.blocks[1].url is None
+async def test_runtime_frontier_starter_url_seed_inspects_session_id_override(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ definition = _FakeDefinition(
+ [
+ _FakeBlock("open", "goto_url", {"url": "https://example.com/search"}),
+ _FakeBlock("search", "navigation", {"url": None}),
+ ]
+ )
+ workflow = _FakeWorkflow(definition)
+ ctx = _make_ctx(browser_session_id="pbs_debug")
+
+ monkeypatch.setattr(
+ tools.app,
+ "PERSISTENT_SESSIONS_MANAGER",
+ _SessionKeyedPersistentSessionsManager(
+ {
+ "pbs_debug": _FakeBrowserState(_FakePage("https://example.com/search/results")),
+ "pbs_fresh_run": _FakeBrowserState(_FakePage("about:blank")),
+ }
+ ),
+ )
+
+ seeded = await tools._workflow_with_runtime_frontier_starter_url_seed(
+ workflow, # type: ignore[arg-type]
+ ctx,
+ labels_to_execute=["search"],
+ runtime_frontier_anchor_url="https://example.com/search",
+ session_id_override="pbs_fresh_run",
+ )
+
+ assert seeded is not workflow
+ assert seeded.workflow_definition.blocks[1].url == "https://example.com/search"
+ assert workflow.workflow_definition.blocks[1].url is None
+
+
@pytest.mark.asyncio
async def test_runtime_frontier_starter_url_seed_preserves_attached_live_page(
monkeypatch: pytest.MonkeyPatch,
@@ -1422,6 +1468,258 @@ def test_run_blocks_outcome_rolls_forward_after_failed_preview() -> None:
assert ctx.last_test_failure_reason is None
+def _recorded_failed_outcome(
+ *,
+ workflow_run_id: str = "wr_fail",
+ block_labels: list[str],
+ attempted_block_label: str,
+ requested_block_labels: list[str] | None = None,
+ workflow_definition: object | None = None,
+) -> RecordedBuildTestOutcome:
+ return RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ attempted_block_label=attempted_block_label,
+ verdict="repairable_failure",
+ reason_code="runtime_block_failure",
+ workflow_run_id=workflow_run_id,
+ block_labels=block_labels,
+ requested_block_labels=requested_block_labels or block_labels,
+ block_shape_hashes=frontier_module._frontier_label_shape_hashes(
+ requested_block_labels or block_labels,
+ workflow_definition,
+ )
+ or {},
+ structural_failure_identity="runtime_failure",
+ )
+
+
+def test_plan_frontier_uses_recorded_failed_block_position_for_same_request_order() -> None:
+ ctx = _make_ctx()
+ definition = _wf_def(
+ ("open", "goto_url", {"url": "https://example.com"}),
+ ("search", "navigation", {"url": None}),
+ ("extract", "extraction", {"prompt": "extract"}),
+ )
+ ctx.latest_recorded_build_test_outcome = _recorded_failed_outcome(
+ block_labels=["open", "search", "extract"],
+ attempted_block_label="search",
+ workflow_definition=definition,
+ )
+
+ labels, seed, frontier = _plan_frontier(
+ ctx,
+ ["open", "search", "extract"],
+ definition,
+ definition,
+ )
+
+ assert labels == ["search", "extract"]
+ assert seed == {}
+ assert frontier == "search"
+
+
+def test_plan_frontier_maps_recorded_failed_block_by_structure_across_label_churn() -> None:
+ ctx = _make_ctx()
+ old = _wf_def(
+ ("open_old", "goto_url", {"url": "https://example.com"}),
+ ("search_old", "navigation", {"url": None}),
+ ("extract_old", "extraction", {"prompt": "extract"}),
+ )
+ new = _wf_def(
+ ("open_new", "goto_url", {"url": "https://example.com"}),
+ ("search_new", "navigation", {"url": None}),
+ ("extract_new", "extraction", {"prompt": "extract"}),
+ )
+ ctx.latest_recorded_build_test_outcome = _recorded_failed_outcome(
+ block_labels=["open_old", "search_old", "extract_old"],
+ attempted_block_label="search_old",
+ workflow_definition=old,
+ )
+
+ labels, seed, frontier = _plan_frontier(
+ ctx,
+ ["open_new", "search_new", "extract_new"],
+ old,
+ new,
+ )
+
+ assert labels == ["search_new", "extract_new"]
+ assert seed == {}
+ assert frontier == "search_new"
+
+
+def test_plan_frontier_fails_closed_when_recorded_failed_order_differs() -> None:
+ ctx = _make_ctx()
+ old = _wf_def(
+ ("open_old", "goto_url", {"url": "https://example.com"}),
+ ("search_old", "navigation", {"url": None}),
+ ("extract_old", "extraction", {"prompt": "extract"}),
+ )
+ new = _wf_def(
+ ("open_new", "goto_url", {"url": "https://example.com"}),
+ ("extract_new", "extraction", {"prompt": "extract"}),
+ ("search_new", "navigation", {"url": None}),
+ )
+ ctx.latest_recorded_build_test_outcome = _recorded_failed_outcome(
+ block_labels=["open_old", "search_old", "extract_old"],
+ attempted_block_label="search_old",
+ workflow_definition=old,
+ )
+
+ labels, seed, frontier = _plan_frontier(
+ ctx,
+ ["open_new", "extract_new", "search_new"],
+ old,
+ new,
+ )
+
+ assert labels == ["open_new", "extract_new", "search_new"]
+ assert seed == {}
+ assert frontier == "open_new"
+
+
+def test_plan_frontier_fails_closed_when_recorded_failed_shapes_are_ambiguous() -> None:
+ ctx = _make_ctx()
+ old = _wf_def(
+ ("first_old", "navigation", {"prompt": "same"}),
+ ("second_old", "navigation", {"prompt": "same"}),
+ ("extract_old", "extraction", {"prompt": "extract"}),
+ )
+ new = _wf_def(
+ ("second_new", "navigation", {"prompt": "same"}),
+ ("first_new", "navigation", {"prompt": "same"}),
+ ("extract_new", "extraction", {"prompt": "extract"}),
+ )
+ ctx.latest_recorded_build_test_outcome = _recorded_failed_outcome(
+ block_labels=["first_old", "second_old", "extract_old"],
+ attempted_block_label="second_old",
+ workflow_definition=old,
+ )
+
+ labels, seed, frontier = _plan_frontier(
+ ctx,
+ ["second_new", "first_new", "extract_new"],
+ old,
+ new,
+ )
+
+ assert labels == ["second_new", "first_new", "extract_new"]
+ assert seed == {}
+ assert frontier == "second_new"
+
+
+def test_plan_frontier_does_not_index_suffix_failed_run_into_full_request() -> None:
+ ctx = _make_ctx()
+ ctx.verified_prefix_labels = ["open_new"]
+ old = _wf_def(
+ ("open_old", "goto_url", {"url": "https://example.com"}),
+ ("search_old", "navigation", {"url": None}),
+ ("extract_old", "extraction", {"prompt": "extract"}),
+ )
+ new = _wf_def(
+ ("open_new", "goto_url", {"url": "https://example.com"}),
+ ("search_new", "navigation", {"url": None}),
+ ("extract_new", "extraction", {"prompt": "extract"}),
+ )
+ ctx.latest_recorded_build_test_outcome = _recorded_failed_outcome(
+ block_labels=["search_old", "extract_old"],
+ attempted_block_label="search_old",
+ requested_block_labels=["open_old", "search_old", "extract_old"],
+ workflow_definition=old,
+ )
+
+ labels, seed, frontier = _plan_frontier(
+ ctx,
+ ["open_new", "search_new", "extract_new"],
+ old,
+ new,
+ )
+
+ assert labels == ["search_new", "extract_new"]
+ assert seed == {}
+ assert frontier == "search_new"
+
+
+def test_recorded_failed_prefix_anchor_maps_relabels_from_recorded_shapes() -> None:
+ ctx = _make_ctx()
+ old = _wf_def(
+ ("open_old", "goto_url", {"url": "https://example.com"}),
+ ("search_old", "navigation", {"url": None}),
+ ("extract_old", "extraction", {"prompt": "extract"}),
+ )
+ definition = _wf_def(
+ ("open_new", "goto_url", {"url": "https://example.com"}),
+ ("search_new", "navigation", {"url": None}),
+ ("extract_new", "extraction", {"prompt": "extract"}),
+ )
+ ctx.latest_recorded_build_test_outcome = _recorded_failed_outcome(
+ block_labels=["open_old", "search_old", "extract_old"],
+ attempted_block_label="search_old",
+ workflow_definition=old,
+ )
+
+ assert frontier_module._has_recorded_failed_prefix_before_frontier(
+ ctx, # type: ignore[arg-type]
+ definition,
+ "search_new",
+ )
+
+
+async def test_recorded_failed_prefix_seeds_fresh_runtime_anchor_without_verified_prefix(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ definition = _FakeDefinition(
+ [
+ _FakeBlock("open", "goto_url", {"url": "https://example.com/login"}),
+ _FakeBlock("search", "navigation", {"url": None}),
+ _FakeBlock("extract", "extraction", {"prompt": "extract"}),
+ ]
+ )
+ workflow = _FakeWorkflow(definition)
+ ctx = _make_ctx(browser_session_id="pbs_debug")
+ ctx.latest_recorded_build_test_outcome = _recorded_failed_outcome(
+ block_labels=["open", "search", "extract"],
+ attempted_block_label="search",
+ workflow_definition=definition,
+ )
+ ctx.workflow_verification_evidence.workflow_run_id = "wr_fail"
+ ctx.workflow_verification_evidence.current_url = "https://example.com/search"
+
+ anchored, anchor_url = tools._workflow_with_runtime_frontier_anchor(
+ workflow, # type: ignore[arg-type]
+ ctx,
+ labels_to_execute=["search", "extract"],
+ frontier_start_label="search",
+ block_outputs_to_seed={},
+ )
+ monkeypatch.setattr(
+ tools.app,
+ "PERSISTENT_SESSIONS_MANAGER",
+ _SessionKeyedPersistentSessionsManager(
+ {
+ "pbs_debug": _FakeBrowserState(_FakePage("https://example.com/search")),
+ "pbs_fresh_run": _FakeBrowserState(_FakePage("about:blank")),
+ }
+ ),
+ )
+
+ seeded = await tools._workflow_with_runtime_frontier_starter_url_seed(
+ anchored, # type: ignore[arg-type]
+ ctx,
+ labels_to_execute=["search", "extract"],
+ runtime_frontier_anchor_url=anchor_url,
+ session_id_override="pbs_fresh_run",
+ )
+
+ assert anchor_url == "https://example.com/search"
+ assert seeded is not workflow
+ assert seeded.workflow_definition.blocks[1].url == "https://example.com/search"
+ assert workflow.workflow_definition.blocks[1].url is None
+ assert ctx.verified_prefix_labels == []
+
+
# --------------------------------------------------------------------------- #
# Edit-time verified-state invalidation #
# --------------------------------------------------------------------------- #