Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-11630: Repair missing output dependency handling (#6993)
Andrew Neilson83edbca03840
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex 913199e76..8a3cbf0b4 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -626,6 +626,17 @@ def _code_authoring_repair_context_prompt(ctx: CopilotContext | None) -> str:f"binding_candidates: {_render_authoring_repair_prompt_list(binding_candidates)}",f"allowed_global_names: {_render_authoring_repair_prompt_list(repair_context.allowed_global_names)}",]+ if repair_context.reason_code == "runtime_missing_output_dependency":+ lines.extend(+ [+ f"missing_output_key: {_clean_authoring_repair_prompt_atom(repair_context.missing_output_key or '')}",+ f"available_output_keys: {_render_authoring_repair_prompt_list(repair_context.available_output_keys)}",+ "output_dependency_failure_class: "+ f"{_clean_authoring_repair_prompt_atom(repair_context.output_dependency_failure_class or '')}",+ "current_block_parameter_keys: "+ f"{_render_authoring_repair_prompt_list(repair_context.current_block_parameter_keys)}",+ ]+ )if repair_context.selector:lines.append(f"selector: {_clean_authoring_repair_prompt_atom(repair_context.selector)}")if repair_context.source_url:@@ -714,6 +725,11 @@ def _code_authoring_repair_context_prompt(ctx: CopilotContext | None) -> str:"For runtime failures, adapt the next code block to the observed page state and do not re-emit ""the same failing selector or name path.")+ if repair_context.reason_code == "runtime_missing_output_dependency":+ lines.append(+ "For missing prior block outputs, bind to an actual available_output_key or repair the producing/current "+ "code block so the output exists; do not create a workflow parameter for missing_output_key."+ )lines.append(_clean_authoring_repair_prompt_atom(repair_context.repair_instruction, max_chars=260))return "\n\n" + "\n".join(line for line in lines if line)diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.pyindex a37b1e49f..01ec33343 100644--- a/skyvern/forge/sdk/copilot/build_test_outcome.py+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py@@ -31,6 +31,7 @@ BuildTestOutcomeVerdict = Literal[BuildTestOutcomeReasonCode = Literal["loaded_result_targets_observed","runtime_block_failure",+ "runtime_missing_output_dependency","sandbox_unresolved_name","synthesized_parameter_binding_ambiguous","code_safety_reject",@@ -186,6 +187,10 @@ def recorded_outcome_from_authoring_repair_context("parameter_keys": sorted(repair_context.parameter_keys),"available_parameter_keys": sorted(repair_context.available_parameter_keys),"binding_candidates": sorted(repair_context.binding_candidates),+ "missing_output_key": _bounded_ref(repair_context.missing_output_key),+ "available_output_keys": sorted(repair_context.available_output_keys),+ "current_block_parameter_keys": sorted(repair_context.current_block_parameter_keys),+ "output_dependency_failure_class": _bounded_ref(repair_context.output_dependency_failure_class),"selector": _bounded_ref(repair_context.selector),"refiner_selector": _bounded_ref(repair_context.refiner_selector),"runtime_failure_class": _bounded_ref(repair_context.runtime_failure_class),@@ -499,6 +504,8 @@ def _authoring_reason_code(value: str) -> BuildTestOutcomeReasonCode:return "synthesized_parameter_binding_ambiguous"if value == "runtime_block_failure":return "runtime_block_failure"+ if value == "runtime_missing_output_dependency":+ return "runtime_missing_output_dependency"if value == "select_option_interaction_mismatch":# Select-option mismatches are author-time policy rejects, not a separate outcome class.return "code_safety_reject"diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex 246ac4189..a03cdeefc 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -164,6 +164,10 @@ class CodeAuthoringRepairContext(BaseModel):allowed_helper_surface: dict[str, list[str]] = Field(default_factory=dict)runtime_failure_reason: str | None = Noneruntime_failure_class: str | None = None+ output_dependency_failure_class: str | None = None+ missing_output_key: str | None = None+ available_output_keys: list[str] = Field(default_factory=list)+ current_block_parameter_keys: list[str] = Field(default_factory=list)failed_block_status: str | None = Noneworkflow_run_id: str | None = Nonecurrent_origin: str | None = Nonediff --git a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.pyindex 3a89f9456..add505051 100644--- a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py+++ b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py@@ -374,6 +374,11 @@ def _repair_context_root_cause_identity(payload["page_result_summaries"] = _safe_identity_list(repair_context.page_result_summaries)payload["page_action_summaries"] = _safe_identity_list(repair_context.page_action_summaries)payload["page_challenge_summaries"] = _safe_identity_list(repair_context.page_challenge_summaries)+ elif reason_code == "runtime_missing_output_dependency":+ payload["missing_output_key"] = _safe_text(repair_context.missing_output_key, 120)+ payload["available_output_keys"] = _safe_identity_list(repair_context.available_output_keys)+ payload["current_block_parameter_keys"] = _safe_identity_list(repair_context.current_block_parameter_keys)+ payload["output_dependency_failure_class"] = _safe_text(repair_context.output_dependency_failure_class, 80)elif repair_context.unresolved_names:payload["unresolved_names"] = _safe_identity_list(repair_context.unresolved_names)diff --git a/skyvern/forge/sdk/copilot/output_contracts.py b/skyvern/forge/sdk/copilot/output_contracts.pynew file mode 100644index 000000000..01a43c53e--- /dev/null+++ b/skyvern/forge/sdk/copilot/output_contracts.py@@ -0,0 +1,159 @@+from __future__ import annotations++from collections.abc import Mapping+from dataclasses import dataclass+from typing import Any++from skyvern.forge.sdk.copilot.workflow_credential_utils import parse_workflow_yaml+from skyvern.forge.sdk.workflow.models.parameter import is_sensitive_workflow_parameter+from skyvern.schemas.workflows import BlockType++_ORDERED_CHILD_BLOCK_LIST_KEYS = ("loop_blocks", "blocks")+_ORDERED_BRANCH_LIST_KEYS = ("branch_conditions", "branches", "ordered_branches")+++@dataclass(frozen=True)+class CodeBlockOutputContract:+ label: str+ code: str+ parameter_keys: tuple[str, ...]+ declared_workflow_parameter_keys: tuple[str, ...]+ available_binding_keys: tuple[str, ...]+ available_output_keys: tuple[str, ...]+++def _enum_or_string_name(value: object) -> str:+ name = getattr(value, "value", value)+ return str(name or "")+++def _block_output_key(block: Mapping[str, Any]) -> str | None:+ label = str(block.get("label") or "").strip()+ return f"{label}_output" if label else None+++def _is_credential_parameter(parameter: Mapping[str, Any]) -> bool:+ parameter_type = str(parameter.get("parameter_type") or "").lower()+ workflow_parameter_type = str(parameter.get("workflow_parameter_type") or "").lower()+ return parameter_type == "credential" or (+ parameter_type == "workflow" and workflow_parameter_type == "credential_id"+ )+++def declared_string_workflow_parameter_keys(parsed: Mapping[str, Any]) -> set[str]:+ workflow_definition = parsed.get("workflow_definition")+ if not isinstance(workflow_definition, Mapping):+ return set()+ parameters = workflow_definition.get("parameters")+ if not isinstance(parameters, list):+ return set()+ keys: set[str] = set()+ for parameter in parameters:+ if not isinstance(parameter, Mapping):+ continue+ key = str(parameter.get("key") or "").strip()+ if not key or _is_credential_parameter(parameter) or is_sensitive_workflow_parameter(dict(parameter)):+ continue+ parameter_type = str(parameter.get("parameter_type") or "").lower()+ workflow_parameter_type = str(parameter.get("workflow_parameter_type") or "").lower()+ if parameter_type and parameter_type != "workflow":+ continue+ if workflow_parameter_type and workflow_parameter_type != "string":+ continue+ keys.add(key)+ return keys+++def declared_workflow_parameter_keys(parsed: Mapping[str, Any]) -> set[str]:+ workflow_definition = parsed.get("workflow_definition")+ if not isinstance(workflow_definition, Mapping):+ return set()+ parameters = workflow_definition.get("parameters")+ if not isinstance(parameters, list):+ return set()+ return {+ key+ for parameter in parameters+ if isinstance(parameter, Mapping)+ for key in [str(parameter.get("key") or "").strip()]+ if key+ }+++def code_block_parameter_keys(block: Mapping[str, Any]) -> frozenset[str]:+ raw_keys = block.get("parameter_keys")+ keys = {key for key in raw_keys if isinstance(key, str) and key} if isinstance(raw_keys, list) else set()+ raw_parameters = block.get("parameters")+ if isinstance(raw_parameters, list):+ keys.update(+ str(parameter.get("key") or "").strip()+ for parameter in raw_parameters+ if isinstance(parameter, Mapping) and str(parameter.get("key") or "").strip()+ )+ return frozenset(keys)+++def code_block_available_contracts_by_label(workflow_yaml: str | None) -> dict[str, CodeBlockOutputContract]:+ if workflow_yaml is None:+ return {}+ parsed = parse_workflow_yaml(workflow_yaml)+ if not isinstance(parsed, Mapping):+ return {}+ contracts: dict[str, CodeBlockOutputContract] = {}+ declared_parameter_keys = tuple(sorted(declared_workflow_parameter_keys(parsed)))++ def visit_branch(+ branch: Mapping[str, Any], available_binding_keys: set[str], available_output_keys: set[str]+ ) -> None:+ for key in _ORDERED_CHILD_BLOCK_LIST_KEYS:+ visit_blocks(branch.get(key), set(available_binding_keys), set(available_output_keys))+ for branch_key in _ORDERED_BRANCH_LIST_KEYS:+ branches = branch.get(branch_key)+ if not isinstance(branches, list):+ continue+ for nested_branch in branches:+ if isinstance(nested_branch, Mapping):+ visit_branch(nested_branch, set(available_binding_keys), set(available_output_keys))++ def visit_blocks(blocks: Any, available_binding_keys: set[str], available_output_keys: set[str]) -> set[str]:+ if not isinstance(blocks, list):+ return available_binding_keys+ for block in blocks:+ if not isinstance(block, Mapping):+ continue+ label = str(block.get("label") or "").strip()+ if label and _enum_or_string_name(block.get("block_type")) == BlockType.CODE.value:+ contracts[label] = CodeBlockOutputContract(+ label=label,+ code=str(block.get("code") or ""),+ parameter_keys=tuple(sorted(code_block_parameter_keys(block))),+ declared_workflow_parameter_keys=declared_parameter_keys,+ available_binding_keys=tuple(sorted(available_binding_keys)),+ available_output_keys=tuple(sorted(available_output_keys)),+ )+ for key in _ORDERED_CHILD_BLOCK_LIST_KEYS:+ visit_blocks(block.get(key), set(available_binding_keys), set(available_output_keys))+ for branch_key in _ORDERED_BRANCH_LIST_KEYS:+ branches = block.get(branch_key)+ if not isinstance(branches, list):+ continue+ for branch in branches:+ if isinstance(branch, Mapping):+ visit_branch(branch, set(available_binding_keys), set(available_output_keys))+ output_key = _block_output_key(block)+ if output_key:+ available_binding_keys.add(output_key)+ available_output_keys.add(output_key)+ return available_binding_keys++ workflow_definition = parsed.get("workflow_definition")+ blocks = workflow_definition.get("blocks") if isinstance(workflow_definition, Mapping) else None+ visit_blocks(blocks, declared_string_workflow_parameter_keys(parsed), set())+ return contracts+++def code_block_available_binding_keys_by_label(workflow_yaml: str | None) -> dict[str, list[str]]:+ return {+ label: list(contract.available_binding_keys)+ for label, contract in code_block_available_contracts_by_label(workflow_yaml).items()+ }diff --git a/skyvern/forge/sdk/copilot/runtime_authoring_repair.py b/skyvern/forge/sdk/copilot/runtime_authoring_repair.pyindex 31aa95b55..9de6e68ce 100644--- a/skyvern/forge/sdk/copilot/runtime_authoring_repair.py+++ b/skyvern/forge/sdk/copilot/runtime_authoring_repair.py@@ -9,6 +9,7 @@ from skyvern.forge.sdk.copilot.composition_evidence import has_bounded_page_schefrom skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, normalize_block_authoring_policyfrom skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContextfrom skyvern.forge.sdk.copilot.failure_tracking import ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY+from skyvern.forge.sdk.copilot.output_contracts import code_block_available_contracts_by_labelfrom skyvern.forge.sdk.copilot.request_policy import redact_raw_secrets_for_promptfrom skyvern.forge.sdk.copilot.run_outcome import trusted_terminal_challenge_category_namefrom skyvern.forge.sdk.copilot.workflow_credential_utils import url_origin@@ -16,16 +17,18 @@ from skyvern.forge.sdk.copilot.workflow_credential_utils import url_originLOG = structlog.get_logger()_RUNTIME_AUTHORING_REASON_CODE = "runtime_block_failure"+_MISSING_OUTPUT_DEPENDENCY_REASON_CODE = "runtime_missing_output_dependency"_RUNTIME_SUMMARY_MAX_CHARS = 120_RUNTIME_SUMMARY_MAX_ITEMS = 5_INSPECT_PAGE_SOURCE_TOOL = "inspect_page_for_composition"+_KEY_ERROR_RE = re.compile(r"KeyError(?:\s*:|\()\s*['\"]([^'\"]+)['\"]")def is_runtime_authoring_repair_context(repair_context: object) -> TypeGuard[CodeAuthoringRepairContext]:- return (- isinstance(repair_context, CodeAuthoringRepairContext)- and repair_context.reason_code == _RUNTIME_AUTHORING_REASON_CODE- )+ return isinstance(repair_context, CodeAuthoringRepairContext) and repair_context.reason_code in {+ _RUNTIME_AUTHORING_REASON_CODE,+ _MISSING_OUTPUT_DEPENDENCY_REASON_CODE,+ }def clear_runtime_authoring_repair_context(copilot_ctx: Any) -> None:@@ -51,6 +54,62 @@ def _runtime_failure_class(reason: str) -> str:return normalized[:80].strip("_") or "runtime_failure"+def _missing_key_from_key_error(reason: str) -> str | None:+ match = _KEY_ERROR_RE.search(reason)+ if match is None:+ return None+ key = match.group(1).strip()+ return key if key else None+++def _missing_output_dependency_context(+ *,+ copilot_ctx: Any,+ block_label: str,+ failed_block_status: str | None,+ failure_reason: str,+ run_id: str,+) -> CodeAuthoringRepairContext | None:+ missing_key = _missing_key_from_key_error(failure_reason)+ workflow_yaml = getattr(copilot_ctx, "workflow_yaml", None)+ if not missing_key or not isinstance(workflow_yaml, str) or not workflow_yaml.strip():+ return None+ contract = code_block_available_contracts_by_label(workflow_yaml).get(block_label)+ if contract is None:+ return None+ if not missing_key.endswith("_output"):+ return None+ if missing_key in contract.available_output_keys:+ return None+ if missing_key in contract.declared_workflow_parameter_keys:+ return None+ if missing_key in contract.available_binding_keys:+ return None+ if missing_key not in contract.parameter_keys:+ return None+ available_output_keys = list(contract.available_output_keys)+ return CodeAuthoringRepairContext(+ block_label=block_label,+ reason_code=_MISSING_OUTPUT_DEPENDENCY_REASON_CODE,+ parameter_keys=list(contract.parameter_keys),+ available_parameter_keys=list(contract.available_binding_keys),+ binding_candidates=available_output_keys,+ runtime_failure_reason=failure_reason,+ runtime_failure_class=_runtime_failure_class(failure_reason),+ output_dependency_failure_class="missing_prior_block_output",+ missing_output_key=missing_key,+ available_output_keys=available_output_keys,+ current_block_parameter_keys=list(contract.parameter_keys),+ failed_block_status=failed_block_status or None,+ workflow_run_id=run_id,+ repair_instruction=(+ "repair the missing prior block output dependency by binding to an actual available prior output key "+ "or changing the producing/current code block so the dependency is real; do not invent a workflow "+ "parameter for this missing output key."+ ),+ )++def _origin_from_runtime_url(value: Any) -> str | None:if not isinstance(value, str) or not value.strip():return None@@ -192,6 +251,16 @@ def record_pending_runtime_authoring_repair_context(copilot_ctx: Any, result: direturnif is_runtime_authoring_repair_context(getattr(copilot_ctx, "last_code_authoring_repair_context", None)):copilot_ctx.last_code_authoring_repair_context = None+ missing_output_context = _missing_output_dependency_context(+ copilot_ctx=copilot_ctx,+ block_label=block_label,+ failed_block_status=failed_block_status or None,+ failure_reason=failure_reason,+ run_id=run_id,+ )+ if missing_output_context is not None:+ copilot_ctx.pending_code_authoring_runtime_repair_context = missing_output_context+ returncopilot_ctx.pending_code_authoring_runtime_repair_context = CodeAuthoringRepairContext(block_label=block_label,reason_code=_RUNTIME_AUTHORING_REASON_CODE,@@ -318,6 +387,8 @@ def finalize_runtime_authoring_repair_context_from_page_observation(pending = getattr(copilot_ctx, "pending_code_authoring_runtime_repair_context", None)if not is_runtime_authoring_repair_context(pending):return None+ if pending.reason_code != _RUNTIME_AUTHORING_REASON_CODE:+ return Noneif not _policy_allows_runtime_authoring_repair(copilot_ctx) or _pending_state_has_stop_or_ask_precedence(copilot_ctx, pending):diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.pyindex ab38380ca..f63488829 100644--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py@@ -74,6 +74,10 @@ from skyvern.forge.sdk.copilot.enforcement import (from skyvern.forge.sdk.copilot.loop_detection import clear_failed_step_tracker_for_tools_in_ctxfrom skyvern.forge.sdk.copilot.narration import CODE_REPAIR_PROGRESS_SURFACE_KIND, CODE_REPAIR_PROGRESS_TEXTfrom skyvern.forge.sdk.copilot.outcome_verification_trace import record_code_artifact_violations+from skyvern.forge.sdk.copilot.output_contracts import (+ code_block_available_binding_keys_by_label,+ declared_string_workflow_parameter_keys,+)from skyvern.forge.sdk.copilot.output_policy import (OutputPolicyReason,OutputPolicyVerdict,@@ -959,79 +963,11 @@ def _unresolved_symbol_repair_context_enabled(ctx: AgentContext) -> bool:def _declared_string_workflow_parameter_keys(parsed: Mapping[str, Any]) -> set[str]:- workflow_definition = parsed.get("workflow_definition")- if not isinstance(workflow_definition, Mapping):- return set()- parameters = workflow_definition.get("parameters")- if not isinstance(parameters, list):- return set()- keys: set[str] = set()- for parameter in parameters:- if not isinstance(parameter, Mapping):- continue- key = str(parameter.get("key") or "").strip()- if not key or _is_credential_parameter(parameter) or is_sensitive_workflow_parameter(dict(parameter)):- continue- parameter_type = str(parameter.get("parameter_type") or "").lower()- workflow_parameter_type = str(parameter.get("workflow_parameter_type") or "").lower()- if parameter_type and parameter_type != "workflow":- continue- if workflow_parameter_type and workflow_parameter_type != "string":- continue- keys.add(key)- return keys+ return declared_string_workflow_parameter_keys(parsed)def _code_block_available_binding_keys_by_label(workflow_yaml: str | None) -> dict[str, list[str]]:- if workflow_yaml is None:- return {}- parsed = parse_workflow_yaml(workflow_yaml)- if not isinstance(parsed, Mapping):- return {}- available_by_label: dict[str, set[str]] = {}-- def block_output_key(block: Mapping[str, Any]) -> str | None:- label = str(block.get("label") or "").strip()- return f"{label}_output" if label else None-- def visit_branch(branch: Mapping[str, Any], available_keys: set[str]) -> None:- for key in _ORDERED_CHILD_BLOCK_LIST_KEYS:- visit_blocks(branch.get(key), set(available_keys))- for branch_key in _ORDERED_BRANCH_LIST_KEYS:- branches = branch.get(branch_key)- if not isinstance(branches, list):- continue- for nested_branch in branches:- if isinstance(nested_branch, Mapping):- visit_branch(nested_branch, set(available_keys))-- def visit_blocks(blocks: Any, available_keys: set[str]) -> set[str]:- if not isinstance(blocks, list):- return available_keys- for block in blocks:- if not isinstance(block, Mapping):- continue- label = str(block.get("label") or "").strip()- if label and _enum_or_string_name(block.get("block_type")) == BlockType.CODE.value:- available_by_label.setdefault(label, set()).update(available_keys)- for key in _ORDERED_CHILD_BLOCK_LIST_KEYS:- visit_blocks(block.get(key), set(available_keys))- for branch_key in _ORDERED_BRANCH_LIST_KEYS:- branches = block.get(branch_key)- if not isinstance(branches, list):- continue- for branch in branches:- if isinstance(branch, Mapping):- visit_branch(branch, set(available_keys))- output_key = block_output_key(block)- if output_key:- available_keys.add(output_key)- return available_keys-- workflow_definition = parsed.get("workflow_definition")- blocks = workflow_definition.get("blocks") if isinstance(workflow_definition, Mapping) else None- visit_blocks(blocks, _declared_string_workflow_parameter_keys(parsed))- return {label: sorted(keys) for label, keys in available_by_label.items()}+ return code_block_available_binding_keys_by_label(workflow_yaml)def _code_block_authoring_repair_context(diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.pyindex f78c8c4d4..d3c68b898 100644--- a/tests/unit/test_copilot_agent_helpers.py+++ b/tests/unit/test_copilot_agent_helpers.py@@ -743,6 +743,35 @@ workflow_definition:assert "do not hardcode the eval value" in promptassert "rerun via update_and_run_blocks" in prompt+ def test_missing_output_dependency_prompt_uses_available_outputs_not_workflow_parameters(self) -> None:+ repair_context = CodeAuthoringRepairContext(+ block_label="read_resource_table",+ reason_code="runtime_missing_output_dependency",+ missing_output_key="create_resource_output",+ available_output_keys=["search_output"],+ current_block_parameter_keys=["create_resource_output"],+ output_dependency_failure_class="missing_prior_block_output",+ repair_instruction=(+ "repair the missing prior block output dependency by binding to an actual available prior output key "+ "or changing the producing/current code block so the dependency is real; do not invent a workflow "+ "parameter for this missing output key."+ ),+ )+ ctx = _ctx(+ block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER,+ last_code_authoring_repair_context=repair_context,+ )++ prompt = agent_module._code_authoring_repair_context_prompt(ctx)++ assert "reason_code: runtime_missing_output_dependency" in prompt+ assert "missing_output_key: create_resource_output" in prompt+ assert "available_output_keys: search_output" in prompt+ assert "current_block_parameter_keys: create_resource_output" in prompt+ assert "bind to an actual available_output_key" in prompt+ assert "do not create a workflow parameter for missing_output_key" in prompt+ assert "create workflow string parameter key create_resource_output" not in prompt+def test_synthesized_parameter_binding_prompt_uses_exact_key(self) -> None:repair_context = CodeAuthoringRepairContext(block_label="order_status",diff --git a/tests/unit/test_copilot_build_test_outcome.py b/tests/unit/test_copilot_build_test_outcome.pyindex a95a2c660..c53c9930e 100644--- a/tests/unit/test_copilot_build_test_outcome.py+++ b/tests/unit/test_copilot_build_test_outcome.py@@ -392,6 +392,33 @@ def test_authoring_repair_context_produces_structural_recorded_outcome() -> None)+def test_authoring_repair_context_missing_output_fields_affect_structural_outcome() -> None:+ context = CodeAuthoringRepairContext(+ block_label="read_resource_table",+ reason_code="runtime_missing_output_dependency",+ missing_output_key="create_resource_output",+ available_output_keys=["search_output"],+ current_block_parameter_keys=["create_resource_output"],+ output_dependency_failure_class="missing_prior_block_output",+ )++ outcome = recorded_outcome_from_authoring_repair_context(context)++ assert outcome.reason_code == "runtime_missing_output_dependency"+ assert (+ outcome.structural_key+ != recorded_outcome_from_authoring_repair_context(+ context.model_copy(update={"missing_output_key": "verify_resource_output"})+ ).structural_key+ )+ assert (+ outcome.structural_key+ != recorded_outcome_from_authoring_repair_context(+ context.model_copy(update={"available_output_keys": ["search_output", "verify_resource_output"]})+ ).structural_key+ )++def test_author_time_reject_structural_payloads_make_distinct_keys() -> None:first = recorded_outcome_from_author_time_reject(reason_code="schema_incompatibility",diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.pyindex b0939e0d2..02b2eaa60 100644--- a/tests/unit/test_copilot_code_block_persist_seam.py+++ b/tests/unit/test_copilot_code_block_persist_seam.py@@ -37,6 +37,7 @@ from skyvern.forge.sdk.copilot.enforcement import (MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS,_check_enforcement,)+from skyvern.forge.sdk.copilot.output_contracts import code_block_available_binding_keys_by_labelfrom skyvern.forge.sdk.copilot.output_utils import sanitize_tool_result_for_llmfrom skyvern.forge.sdk.copilot.reached_download_target import ReachedDownloadTargetfrom skyvern.forge.sdk.copilot.request_policy import RequestPolicy@@ -2088,6 +2089,42 @@ class TestCodeBlockParameterPersistSeam:== "Workflow YAML nesting is too deep to validate.")+ def test_output_contract_graph_preserves_branch_isolation(self) -> None:+ workflow_yaml = _yaml(+ """+ workflow_definition:+ blocks:+ - block_type: code+ label: start_search+ code: |+ return {"ok": True}+ - block_type: task+ label: choose_path+ branch_conditions:+ - blocks:+ - block_type: code+ label: branch_a+ code: |+ return start_search_output+ - blocks:+ - block_type: code+ label: branch_b+ code: |+ return start_search_output+ - block_type: code+ label: after_choice+ code: |+ return choose_path_output+ """+ )++ available_by_label = code_block_available_binding_keys_by_label(workflow_yaml)++ assert available_by_label["branch_a"] == ["start_search_output"]+ assert available_by_label["branch_b"] == ["start_search_output"]+ assert "branch_a_output" not in available_by_label["branch_b"]+ assert available_by_label["after_choice"] == ["choose_path_output", "start_search_output"]+@pytest.mark.asyncioasync def test_declared_workflow_string_parameter_key_is_accepted(self, monkeypatch: pytest.MonkeyPatch) -> None:_stub_successful_update(monkeypatch)diff --git a/tests/unit/test_copilot_diagnosis_repair_contract.py b/tests/unit/test_copilot_diagnosis_repair_contract.pyindex 409a513dc..7580f4e1a 100644--- a/tests/unit/test_copilot_diagnosis_repair_contract.py+++ b/tests/unit/test_copilot_diagnosis_repair_contract.py@@ -28,6 +28,7 @@ from skyvern.forge.sdk.copilot.runtime_authoring_repair import (finalize_runtime_authoring_repair_context_from_page_observation,inject_runtime_authoring_repair_context,post_run_inspection_cleanly_matches,+ record_pending_runtime_authoring_repair_context,)from skyvern.forge.sdk.copilot.tools import run_execution as run_execution_modulefrom skyvern.forge.sdk.copilot.tools.composition_capture import store_post_run_page_evidence@@ -51,6 +52,72 @@ def _ctx() -> CopilotContext:)+def _runtime_output_dependency_yaml(*, available: bool = False) -> str:+ producer_label = "create_or_verify_resource" if available else "create_resource"+ return f"""+workflow_definition:+ blocks:+ - block_type: code+ label: {producer_label}+ code: |+ return {{"ok": True}}+ - block_type: code+ label: read_resource_table+ parameter_keys: [create_or_verify_resource_output]+ code: |+ resource = create_or_verify_resource_output["id"]+ return {{"resource": resource}}+"""+++def _runtime_declared_output_named_input_yaml() -> str:+ return """+workflow_definition:+ parameters:+ - key: create_or_verify_resource_output+ parameter_type: workflow+ workflow_parameter_type: string+ blocks:+ - block_type: code+ label: read_resource_table+ parameter_keys: [create_or_verify_resource_output]+ code: |+ resource = create_or_verify_resource_output["id"]+ return {"resource": resource}+"""+++def _runtime_declared_non_string_output_named_input_yaml() -> str:+ return """+workflow_definition:+ parameters:+ - key: create_or_verify_resource_output+ parameter_type: workflow+ workflow_parameter_type: number+ blocks:+ - block_type: code+ label: read_resource_table+ parameter_keys: [create_or_verify_resource_output]+ code: |+ resource = create_or_verify_resource_output["id"]+ return {"resource": resource}+"""+++def _runtime_output_substring_only_yaml() -> str:+ return """+workflow_definition:+ blocks:+ - block_type: code+ label: read_resource_table+ code: |+ # foo_output appears in prose only.+ data = {"foo_output": {"id": "fixture"}}+ literal = "foo_output"+ return data["foo_output"]+"""++def _satisfied_completion_verification() -> CompletionVerificationResult:return CompletionVerificationResult(status="evaluated",@@ -245,6 +312,45 @@ def test_missing_required_output_key_repair_identity_uses_structural_context_onlassert contract.diagnosis_result.root_cause_identity.error_class == "code_authoring_missing_required_output_key"+def test_runtime_missing_output_dependency_identity_uses_key_and_available_contracts() -> None:+ base = CodeAuthoringRepairContext(+ block_label="read_resource_table",+ reason_code="runtime_missing_output_dependency",+ missing_output_key="create_resource_output",+ available_output_keys=["search_output", "verify_output"],+ current_block_parameter_keys=["create_resource_output"],+ output_dependency_failure_class="missing_prior_block_output",+ )+ reordered = base.model_copy(update={"available_output_keys": ["verify_output", "search_output"]})+ different_key = base.model_copy(update={"missing_output_key": "verify_resource_output"})+ different_available = base.model_copy(update={"available_output_keys": ["search_output"]})++ base_signature = build_diagnosis_repair_contract(+ source_tool="update_and_run_blocks",+ result=_authoring_repair_result(base),+ ctx=_ctx(),+ ).to_trace_data()["root_cause_signature"]+ reordered_signature = build_diagnosis_repair_contract(+ source_tool="update_and_run_blocks",+ result=_authoring_repair_result(reordered),+ ctx=_ctx(),+ ).to_trace_data()["root_cause_signature"]+ different_key_signature = build_diagnosis_repair_contract(+ source_tool="update_and_run_blocks",+ result=_authoring_repair_result(different_key),+ ctx=_ctx(),+ ).to_trace_data()["root_cause_signature"]+ different_available_signature = build_diagnosis_repair_contract(+ source_tool="update_and_run_blocks",+ result=_authoring_repair_result(different_available),+ ctx=_ctx(),+ ).to_trace_data()["root_cause_signature"]++ assert base_signature == reordered_signature+ assert different_key_signature != base_signature+ assert different_available_signature != base_signature++def test_runtime_authoring_repair_context_identity_includes_bounded_page_state() -> None:base = CodeAuthoringRepairContext(block_label="search_registry",@@ -431,6 +537,153 @@ def test_failed_run_injects_pending_runtime_authoring_context_before_page_observassert contract.repair_decision.target_blocks == ["search_registry"]+def test_runtime_key_error_for_missing_prior_output_records_typed_authoring_context() -> None:+ ctx = _ctx()+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER+ ctx.workflow_yaml = _runtime_output_dependency_yaml(available=False)+ result = {+ "ok": False,+ "error": "Run failed.",+ "data": {+ "workflow_run_id": "wr_missing_output",+ "overall_status": "failed",+ "blocks": [+ {+ "label": "read_resource_table",+ "status": "failed",+ "failure_reason": "KeyError: 'create_or_verify_resource_output'",+ }+ ],+ },+ }++ record_pending_runtime_authoring_repair_context(ctx, result)+ inject_runtime_authoring_repair_context(ctx, result)++ repair_context = ctx.last_code_authoring_repair_context+ assert isinstance(repair_context, CodeAuthoringRepairContext)+ assert repair_context.reason_code == "runtime_missing_output_dependency"+ assert repair_context.block_label == "read_resource_table"+ assert repair_context.workflow_run_id == "wr_missing_output"+ assert repair_context.output_dependency_failure_class == "missing_prior_block_output"+ assert repair_context.missing_output_key == "create_or_verify_resource_output"+ assert repair_context.available_output_keys == ["create_resource_output"]+ assert repair_context.current_block_parameter_keys == ["create_or_verify_resource_output"]+ assert result["data"]["authoring_repair_context"] == repair_context.model_dump(mode="json")+++def test_runtime_key_error_for_available_prior_output_keeps_generic_runtime_repair() -> None:+ ctx = _ctx()+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER+ ctx.workflow_yaml = _runtime_output_dependency_yaml(available=True)+ result = {+ "ok": False,+ "error": "Run failed.",+ "data": {+ "workflow_run_id": "wr_available_output",+ "overall_status": "failed",+ "blocks": [+ {+ "label": "read_resource_table",+ "status": "failed",+ "failure_reason": "KeyError: 'create_or_verify_resource_output'",+ }+ ],+ },+ }++ record_pending_runtime_authoring_repair_context(ctx, result)++ pending_context = ctx.pending_code_authoring_runtime_repair_context+ assert isinstance(pending_context, CodeAuthoringRepairContext)+ assert pending_context.reason_code == "runtime_block_failure"+ assert pending_context.missing_output_key is None+++def test_runtime_key_error_for_declared_workflow_input_keeps_generic_runtime_repair() -> None:+ ctx = _ctx()+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER+ ctx.workflow_yaml = _runtime_declared_output_named_input_yaml()+ result = {+ "ok": False,+ "error": "Run failed.",+ "data": {+ "workflow_run_id": "wr_declared_input",+ "overall_status": "failed",+ "blocks": [+ {+ "label": "read_resource_table",+ "status": "failed",+ "failure_reason": "KeyError: 'create_or_verify_resource_output'",+ }+ ],+ },+ }++ record_pending_runtime_authoring_repair_context(ctx, result)++ pending_context = ctx.pending_code_authoring_runtime_repair_context+ assert isinstance(pending_context, CodeAuthoringRepairContext)+ assert pending_context.reason_code == "runtime_block_failure"+ assert pending_context.missing_output_key is None+++def test_runtime_key_error_for_declared_non_string_workflow_input_keeps_generic_runtime_repair() -> None:+ ctx = _ctx()+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER+ ctx.workflow_yaml = _runtime_declared_non_string_output_named_input_yaml()+ result = {+ "ok": False,+ "error": "Run failed.",+ "data": {+ "workflow_run_id": "wr_declared_number_input",+ "overall_status": "failed",+ "blocks": [+ {+ "label": "read_resource_table",+ "status": "failed",+ "failure_reason": "KeyError: 'create_or_verify_resource_output'",+ }+ ],+ },+ }++ record_pending_runtime_authoring_repair_context(ctx, result)++ pending_context = ctx.pending_code_authoring_runtime_repair_context+ assert isinstance(pending_context, CodeAuthoringRepairContext)+ assert pending_context.reason_code == "runtime_block_failure"+ assert pending_context.missing_output_key is None+++def test_runtime_key_error_for_code_substring_only_keeps_generic_runtime_repair() -> None:+ ctx = _ctx()+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER+ ctx.workflow_yaml = _runtime_output_substring_only_yaml()+ result = {+ "ok": False,+ "error": "Run failed.",+ "data": {+ "workflow_run_id": "wr_substring_only",+ "overall_status": "failed",+ "blocks": [+ {+ "label": "read_resource_table",+ "status": "failed",+ "failure_reason": "KeyError: 'foo_output'",+ }+ ],+ },+ }++ record_pending_runtime_authoring_repair_context(ctx, result)++ pending_context = ctx.pending_code_authoring_runtime_repair_context+ assert isinstance(pending_context, CodeAuthoringRepairContext)+ assert pending_context.reason_code == "runtime_block_failure"+ assert pending_context.missing_output_key is None++def _injected_repair_log(events: list[dict[str, object]]) -> dict[str, object]:matches = [event for event in events if event.get("event") == "Injected runtime authoring repair context"]assert len(matches) == 1