Skip to content

Automate browser based workflows with AI

SKY-11630: Repair missing output dependency handling (#6993)

Andrew Neilson83edbca03840
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 913199e76..8a3cbf0b4 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -626,6 +626,17 @@ def _code_authoring_repair_context_prompt(ctx: CopilotContext | None) -> str:
f"binding_candidates: {_render_authoring_repair_prompt_list(binding_candidates)}",
f"allowed_global_names: {_render_authoring_repair_prompt_list(repair_context.allowed_global_names)}",
]
+ if repair_context.reason_code == "runtime_missing_output_dependency":
+ lines.extend(
+ [
+ f"missing_output_key: {_clean_authoring_repair_prompt_atom(repair_context.missing_output_key or '')}",
+ f"available_output_keys: {_render_authoring_repair_prompt_list(repair_context.available_output_keys)}",
+ "output_dependency_failure_class: "
+ f"{_clean_authoring_repair_prompt_atom(repair_context.output_dependency_failure_class or '')}",
+ "current_block_parameter_keys: "
+ f"{_render_authoring_repair_prompt_list(repair_context.current_block_parameter_keys)}",
+ ]
+ )
if repair_context.selector:
lines.append(f"selector: {_clean_authoring_repair_prompt_atom(repair_context.selector)}")
if repair_context.source_url:
@@ -714,6 +725,11 @@ def _code_authoring_repair_context_prompt(ctx: CopilotContext | None) -> str:
"For runtime failures, adapt the next code block to the observed page state and do not re-emit "
"the same failing selector or name path."
)
+ if repair_context.reason_code == "runtime_missing_output_dependency":
+ lines.append(
+ "For missing prior block outputs, bind to an actual available_output_key or repair the producing/current "
+ "code block so the output exists; do not create a workflow parameter for missing_output_key."
+ )
lines.append(_clean_authoring_repair_prompt_atom(repair_context.repair_instruction, max_chars=260))
return "\n\n" + "\n".join(line for line in lines if line)
diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.py
index a37b1e49f..01ec33343 100644
--- a/skyvern/forge/sdk/copilot/build_test_outcome.py
+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py
@@ -31,6 +31,7 @@ BuildTestOutcomeVerdict = Literal[
BuildTestOutcomeReasonCode = Literal[
"loaded_result_targets_observed",
"runtime_block_failure",
+ "runtime_missing_output_dependency",
"sandbox_unresolved_name",
"synthesized_parameter_binding_ambiguous",
"code_safety_reject",
@@ -186,6 +187,10 @@ def recorded_outcome_from_authoring_repair_context(
"parameter_keys": sorted(repair_context.parameter_keys),
"available_parameter_keys": sorted(repair_context.available_parameter_keys),
"binding_candidates": sorted(repair_context.binding_candidates),
+ "missing_output_key": _bounded_ref(repair_context.missing_output_key),
+ "available_output_keys": sorted(repair_context.available_output_keys),
+ "current_block_parameter_keys": sorted(repair_context.current_block_parameter_keys),
+ "output_dependency_failure_class": _bounded_ref(repair_context.output_dependency_failure_class),
"selector": _bounded_ref(repair_context.selector),
"refiner_selector": _bounded_ref(repair_context.refiner_selector),
"runtime_failure_class": _bounded_ref(repair_context.runtime_failure_class),
@@ -499,6 +504,8 @@ def _authoring_reason_code(value: str) -> BuildTestOutcomeReasonCode:
return "synthesized_parameter_binding_ambiguous"
if value == "runtime_block_failure":
return "runtime_block_failure"
+ if value == "runtime_missing_output_dependency":
+ return "runtime_missing_output_dependency"
if value == "select_option_interaction_mismatch":
# Select-option mismatches are author-time policy rejects, not a separate outcome class.
return "code_safety_reject"
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index 246ac4189..a03cdeefc 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -164,6 +164,10 @@ class CodeAuthoringRepairContext(BaseModel):
allowed_helper_surface: dict[str, list[str]] = Field(default_factory=dict)
runtime_failure_reason: str | None = None
runtime_failure_class: str | None = None
+ output_dependency_failure_class: str | None = None
+ missing_output_key: str | None = None
+ available_output_keys: list[str] = Field(default_factory=list)
+ current_block_parameter_keys: list[str] = Field(default_factory=list)
failed_block_status: str | None = None
workflow_run_id: str | None = None
current_origin: str | None = None
diff --git a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
index 3a89f9456..add505051 100644
--- a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
+++ b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
@@ -374,6 +374,11 @@ def _repair_context_root_cause_identity(
payload["page_result_summaries"] = _safe_identity_list(repair_context.page_result_summaries)
payload["page_action_summaries"] = _safe_identity_list(repair_context.page_action_summaries)
payload["page_challenge_summaries"] = _safe_identity_list(repair_context.page_challenge_summaries)
+ elif reason_code == "runtime_missing_output_dependency":
+ payload["missing_output_key"] = _safe_text(repair_context.missing_output_key, 120)
+ payload["available_output_keys"] = _safe_identity_list(repair_context.available_output_keys)
+ payload["current_block_parameter_keys"] = _safe_identity_list(repair_context.current_block_parameter_keys)
+ payload["output_dependency_failure_class"] = _safe_text(repair_context.output_dependency_failure_class, 80)
elif repair_context.unresolved_names:
payload["unresolved_names"] = _safe_identity_list(repair_context.unresolved_names)
diff --git a/skyvern/forge/sdk/copilot/output_contracts.py b/skyvern/forge/sdk/copilot/output_contracts.py
new file mode 100644
index 000000000..01a43c53e
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/output_contracts.py
@@ -0,0 +1,159 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Any
+
+from skyvern.forge.sdk.copilot.workflow_credential_utils import parse_workflow_yaml
+from skyvern.forge.sdk.workflow.models.parameter import is_sensitive_workflow_parameter
+from skyvern.schemas.workflows import BlockType
+
+_ORDERED_CHILD_BLOCK_LIST_KEYS = ("loop_blocks", "blocks")
+_ORDERED_BRANCH_LIST_KEYS = ("branch_conditions", "branches", "ordered_branches")
+
+
+@dataclass(frozen=True)
+class CodeBlockOutputContract:
+ label: str
+ code: str
+ parameter_keys: tuple[str, ...]
+ declared_workflow_parameter_keys: tuple[str, ...]
+ available_binding_keys: tuple[str, ...]
+ available_output_keys: tuple[str, ...]
+
+
+def _enum_or_string_name(value: object) -> str:
+ name = getattr(value, "value", value)
+ return str(name or "")
+
+
+def _block_output_key(block: Mapping[str, Any]) -> str | None:
+ label = str(block.get("label") or "").strip()
+ return f"{label}_output" if label else None
+
+
+def _is_credential_parameter(parameter: Mapping[str, Any]) -> bool:
+ parameter_type = str(parameter.get("parameter_type") or "").lower()
+ workflow_parameter_type = str(parameter.get("workflow_parameter_type") or "").lower()
+ return parameter_type == "credential" or (
+ parameter_type == "workflow" and workflow_parameter_type == "credential_id"
+ )
+
+
+def declared_string_workflow_parameter_keys(parsed: Mapping[str, Any]) -> set[str]:
+ workflow_definition = parsed.get("workflow_definition")
+ if not isinstance(workflow_definition, Mapping):
+ return set()
+ parameters = workflow_definition.get("parameters")
+ if not isinstance(parameters, list):
+ return set()
+ keys: set[str] = set()
+ for parameter in parameters:
+ if not isinstance(parameter, Mapping):
+ continue
+ key = str(parameter.get("key") or "").strip()
+ if not key or _is_credential_parameter(parameter) or is_sensitive_workflow_parameter(dict(parameter)):
+ continue
+ parameter_type = str(parameter.get("parameter_type") or "").lower()
+ workflow_parameter_type = str(parameter.get("workflow_parameter_type") or "").lower()
+ if parameter_type and parameter_type != "workflow":
+ continue
+ if workflow_parameter_type and workflow_parameter_type != "string":
+ continue
+ keys.add(key)
+ return keys
+
+
+def declared_workflow_parameter_keys(parsed: Mapping[str, Any]) -> set[str]:
+ workflow_definition = parsed.get("workflow_definition")
+ if not isinstance(workflow_definition, Mapping):
+ return set()
+ parameters = workflow_definition.get("parameters")
+ if not isinstance(parameters, list):
+ return set()
+ return {
+ key
+ for parameter in parameters
+ if isinstance(parameter, Mapping)
+ for key in [str(parameter.get("key") or "").strip()]
+ if key
+ }
+
+
+def code_block_parameter_keys(block: Mapping[str, Any]) -> frozenset[str]:
+ raw_keys = block.get("parameter_keys")
+ keys = {key for key in raw_keys if isinstance(key, str) and key} if isinstance(raw_keys, list) else set()
+ raw_parameters = block.get("parameters")
+ if isinstance(raw_parameters, list):
+ keys.update(
+ str(parameter.get("key") or "").strip()
+ for parameter in raw_parameters
+ if isinstance(parameter, Mapping) and str(parameter.get("key") or "").strip()
+ )
+ return frozenset(keys)
+
+
+def code_block_available_contracts_by_label(workflow_yaml: str | None) -> dict[str, CodeBlockOutputContract]:
+ if workflow_yaml is None:
+ return {}
+ parsed = parse_workflow_yaml(workflow_yaml)
+ if not isinstance(parsed, Mapping):
+ return {}
+ contracts: dict[str, CodeBlockOutputContract] = {}
+ declared_parameter_keys = tuple(sorted(declared_workflow_parameter_keys(parsed)))
+
+ def visit_branch(
+ branch: Mapping[str, Any], available_binding_keys: set[str], available_output_keys: set[str]
+ ) -> None:
+ for key in _ORDERED_CHILD_BLOCK_LIST_KEYS:
+ visit_blocks(branch.get(key), set(available_binding_keys), set(available_output_keys))
+ for branch_key in _ORDERED_BRANCH_LIST_KEYS:
+ branches = branch.get(branch_key)
+ if not isinstance(branches, list):
+ continue
+ for nested_branch in branches:
+ if isinstance(nested_branch, Mapping):
+ visit_branch(nested_branch, set(available_binding_keys), set(available_output_keys))
+
+ def visit_blocks(blocks: Any, available_binding_keys: set[str], available_output_keys: set[str]) -> set[str]:
+ if not isinstance(blocks, list):
+ return available_binding_keys
+ for block in blocks:
+ if not isinstance(block, Mapping):
+ continue
+ label = str(block.get("label") or "").strip()
+ if label and _enum_or_string_name(block.get("block_type")) == BlockType.CODE.value:
+ contracts[label] = CodeBlockOutputContract(
+ label=label,
+ code=str(block.get("code") or ""),
+ parameter_keys=tuple(sorted(code_block_parameter_keys(block))),
+ declared_workflow_parameter_keys=declared_parameter_keys,
+ available_binding_keys=tuple(sorted(available_binding_keys)),
+ available_output_keys=tuple(sorted(available_output_keys)),
+ )
+ for key in _ORDERED_CHILD_BLOCK_LIST_KEYS:
+ visit_blocks(block.get(key), set(available_binding_keys), set(available_output_keys))
+ for branch_key in _ORDERED_BRANCH_LIST_KEYS:
+ branches = block.get(branch_key)
+ if not isinstance(branches, list):
+ continue
+ for branch in branches:
+ if isinstance(branch, Mapping):
+ visit_branch(branch, set(available_binding_keys), set(available_output_keys))
+ output_key = _block_output_key(block)
+ if output_key:
+ available_binding_keys.add(output_key)
+ available_output_keys.add(output_key)
+ return available_binding_keys
+
+ workflow_definition = parsed.get("workflow_definition")
+ blocks = workflow_definition.get("blocks") if isinstance(workflow_definition, Mapping) else None
+ visit_blocks(blocks, declared_string_workflow_parameter_keys(parsed), set())
+ return contracts
+
+
+def code_block_available_binding_keys_by_label(workflow_yaml: str | None) -> dict[str, list[str]]:
+ return {
+ label: list(contract.available_binding_keys)
+ for label, contract in code_block_available_contracts_by_label(workflow_yaml).items()
+ }
diff --git a/skyvern/forge/sdk/copilot/runtime_authoring_repair.py b/skyvern/forge/sdk/copilot/runtime_authoring_repair.py
index 31aa95b55..9de6e68ce 100644
--- a/skyvern/forge/sdk/copilot/runtime_authoring_repair.py
+++ b/skyvern/forge/sdk/copilot/runtime_authoring_repair.py
@@ -9,6 +9,7 @@ from skyvern.forge.sdk.copilot.composition_evidence import has_bounded_page_sche
from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, normalize_block_authoring_policy
from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext
from skyvern.forge.sdk.copilot.failure_tracking import ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY
+from skyvern.forge.sdk.copilot.output_contracts import code_block_available_contracts_by_label
from skyvern.forge.sdk.copilot.request_policy import redact_raw_secrets_for_prompt
from skyvern.forge.sdk.copilot.run_outcome import trusted_terminal_challenge_category_name
from skyvern.forge.sdk.copilot.workflow_credential_utils import url_origin
@@ -16,16 +17,18 @@ from skyvern.forge.sdk.copilot.workflow_credential_utils import url_origin
LOG = structlog.get_logger()
_RUNTIME_AUTHORING_REASON_CODE = "runtime_block_failure"
+_MISSING_OUTPUT_DEPENDENCY_REASON_CODE = "runtime_missing_output_dependency"
_RUNTIME_SUMMARY_MAX_CHARS = 120
_RUNTIME_SUMMARY_MAX_ITEMS = 5
_INSPECT_PAGE_SOURCE_TOOL = "inspect_page_for_composition"
+_KEY_ERROR_RE = re.compile(r"KeyError(?:\s*:|\()\s*['\"]([^'\"]+)['\"]")
def is_runtime_authoring_repair_context(repair_context: object) -> TypeGuard[CodeAuthoringRepairContext]:
- return (
- isinstance(repair_context, CodeAuthoringRepairContext)
- and repair_context.reason_code == _RUNTIME_AUTHORING_REASON_CODE
- )
+ return isinstance(repair_context, CodeAuthoringRepairContext) and repair_context.reason_code in {
+ _RUNTIME_AUTHORING_REASON_CODE,
+ _MISSING_OUTPUT_DEPENDENCY_REASON_CODE,
+ }
def clear_runtime_authoring_repair_context(copilot_ctx: Any) -> None:
@@ -51,6 +54,62 @@ def _runtime_failure_class(reason: str) -> str:
return normalized[:80].strip("_") or "runtime_failure"
+def _missing_key_from_key_error(reason: str) -> str | None:
+ match = _KEY_ERROR_RE.search(reason)
+ if match is None:
+ return None
+ key = match.group(1).strip()
+ return key if key else None
+
+
+def _missing_output_dependency_context(
+ *,
+ copilot_ctx: Any,
+ block_label: str,
+ failed_block_status: str | None,
+ failure_reason: str,
+ run_id: str,
+) -> CodeAuthoringRepairContext | None:
+ missing_key = _missing_key_from_key_error(failure_reason)
+ workflow_yaml = getattr(copilot_ctx, "workflow_yaml", None)
+ if not missing_key or not isinstance(workflow_yaml, str) or not workflow_yaml.strip():
+ return None
+ contract = code_block_available_contracts_by_label(workflow_yaml).get(block_label)
+ if contract is None:
+ return None
+ if not missing_key.endswith("_output"):
+ return None
+ if missing_key in contract.available_output_keys:
+ return None
+ if missing_key in contract.declared_workflow_parameter_keys:
+ return None
+ if missing_key in contract.available_binding_keys:
+ return None
+ if missing_key not in contract.parameter_keys:
+ return None
+ available_output_keys = list(contract.available_output_keys)
+ return CodeAuthoringRepairContext(
+ block_label=block_label,
+ reason_code=_MISSING_OUTPUT_DEPENDENCY_REASON_CODE,
+ parameter_keys=list(contract.parameter_keys),
+ available_parameter_keys=list(contract.available_binding_keys),
+ binding_candidates=available_output_keys,
+ runtime_failure_reason=failure_reason,
+ runtime_failure_class=_runtime_failure_class(failure_reason),
+ output_dependency_failure_class="missing_prior_block_output",
+ missing_output_key=missing_key,
+ available_output_keys=available_output_keys,
+ current_block_parameter_keys=list(contract.parameter_keys),
+ failed_block_status=failed_block_status or None,
+ workflow_run_id=run_id,
+ repair_instruction=(
+ "repair the missing prior block output dependency by binding to an actual available prior output key "
+ "or changing the producing/current code block so the dependency is real; do not invent a workflow "
+ "parameter for this missing output key."
+ ),
+ )
+
+
def _origin_from_runtime_url(value: Any) -> str | None:
if not isinstance(value, str) or not value.strip():
return None
@@ -192,6 +251,16 @@ def record_pending_runtime_authoring_repair_context(copilot_ctx: Any, result: di
return
if is_runtime_authoring_repair_context(getattr(copilot_ctx, "last_code_authoring_repair_context", None)):
copilot_ctx.last_code_authoring_repair_context = None
+ missing_output_context = _missing_output_dependency_context(
+ copilot_ctx=copilot_ctx,
+ block_label=block_label,
+ failed_block_status=failed_block_status or None,
+ failure_reason=failure_reason,
+ run_id=run_id,
+ )
+ if missing_output_context is not None:
+ copilot_ctx.pending_code_authoring_runtime_repair_context = missing_output_context
+ return
copilot_ctx.pending_code_authoring_runtime_repair_context = CodeAuthoringRepairContext(
block_label=block_label,
reason_code=_RUNTIME_AUTHORING_REASON_CODE,
@@ -318,6 +387,8 @@ def finalize_runtime_authoring_repair_context_from_page_observation(
pending = getattr(copilot_ctx, "pending_code_authoring_runtime_repair_context", None)
if not is_runtime_authoring_repair_context(pending):
return None
+ if pending.reason_code != _RUNTIME_AUTHORING_REASON_CODE:
+ return None
if not _policy_allows_runtime_authoring_repair(copilot_ctx) or _pending_state_has_stop_or_ask_precedence(
copilot_ctx, pending
):
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index ab38380ca..f63488829 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -74,6 +74,10 @@ from skyvern.forge.sdk.copilot.enforcement import (
from skyvern.forge.sdk.copilot.loop_detection import clear_failed_step_tracker_for_tools_in_ctx
from skyvern.forge.sdk.copilot.narration import CODE_REPAIR_PROGRESS_SURFACE_KIND, CODE_REPAIR_PROGRESS_TEXT
from skyvern.forge.sdk.copilot.outcome_verification_trace import record_code_artifact_violations
+from skyvern.forge.sdk.copilot.output_contracts import (
+ code_block_available_binding_keys_by_label,
+ declared_string_workflow_parameter_keys,
+)
from skyvern.forge.sdk.copilot.output_policy import (
OutputPolicyReason,
OutputPolicyVerdict,
@@ -959,79 +963,11 @@ def _unresolved_symbol_repair_context_enabled(ctx: AgentContext) -> bool:
def _declared_string_workflow_parameter_keys(parsed: Mapping[str, Any]) -> set[str]:
- workflow_definition = parsed.get("workflow_definition")
- if not isinstance(workflow_definition, Mapping):
- return set()
- parameters = workflow_definition.get("parameters")
- if not isinstance(parameters, list):
- return set()
- keys: set[str] = set()
- for parameter in parameters:
- if not isinstance(parameter, Mapping):
- continue
- key = str(parameter.get("key") or "").strip()
- if not key or _is_credential_parameter(parameter) or is_sensitive_workflow_parameter(dict(parameter)):
- continue
- parameter_type = str(parameter.get("parameter_type") or "").lower()
- workflow_parameter_type = str(parameter.get("workflow_parameter_type") or "").lower()
- if parameter_type and parameter_type != "workflow":
- continue
- if workflow_parameter_type and workflow_parameter_type != "string":
- continue
- keys.add(key)
- return keys
+ return declared_string_workflow_parameter_keys(parsed)
def _code_block_available_binding_keys_by_label(workflow_yaml: str | None) -> dict[str, list[str]]:
- if workflow_yaml is None:
- return {}
- parsed = parse_workflow_yaml(workflow_yaml)
- if not isinstance(parsed, Mapping):
- return {}
- available_by_label: dict[str, set[str]] = {}
-
- def block_output_key(block: Mapping[str, Any]) -> str | None:
- label = str(block.get("label") or "").strip()
- return f"{label}_output" if label else None
-
- def visit_branch(branch: Mapping[str, Any], available_keys: set[str]) -> None:
- for key in _ORDERED_CHILD_BLOCK_LIST_KEYS:
- visit_blocks(branch.get(key), set(available_keys))
- for branch_key in _ORDERED_BRANCH_LIST_KEYS:
- branches = branch.get(branch_key)
- if not isinstance(branches, list):
- continue
- for nested_branch in branches:
- if isinstance(nested_branch, Mapping):
- visit_branch(nested_branch, set(available_keys))
-
- def visit_blocks(blocks: Any, available_keys: set[str]) -> set[str]:
- if not isinstance(blocks, list):
- return available_keys
- for block in blocks:
- if not isinstance(block, Mapping):
- continue
- label = str(block.get("label") or "").strip()
- if label and _enum_or_string_name(block.get("block_type")) == BlockType.CODE.value:
- available_by_label.setdefault(label, set()).update(available_keys)
- for key in _ORDERED_CHILD_BLOCK_LIST_KEYS:
- visit_blocks(block.get(key), set(available_keys))
- for branch_key in _ORDERED_BRANCH_LIST_KEYS:
- branches = block.get(branch_key)
- if not isinstance(branches, list):
- continue
- for branch in branches:
- if isinstance(branch, Mapping):
- visit_branch(branch, set(available_keys))
- output_key = block_output_key(block)
- if output_key:
- available_keys.add(output_key)
- return available_keys
-
- workflow_definition = parsed.get("workflow_definition")
- blocks = workflow_definition.get("blocks") if isinstance(workflow_definition, Mapping) else None
- visit_blocks(blocks, _declared_string_workflow_parameter_keys(parsed))
- return {label: sorted(keys) for label, keys in available_by_label.items()}
+ return code_block_available_binding_keys_by_label(workflow_yaml)
def _code_block_authoring_repair_context(
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index f78c8c4d4..d3c68b898 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -743,6 +743,35 @@ workflow_definition:
assert "do not hardcode the eval value" in prompt
assert "rerun via update_and_run_blocks" in prompt
+ def test_missing_output_dependency_prompt_uses_available_outputs_not_workflow_parameters(self) -> None:
+ repair_context = CodeAuthoringRepairContext(
+ block_label="read_resource_table",
+ reason_code="runtime_missing_output_dependency",
+ missing_output_key="create_resource_output",
+ available_output_keys=["search_output"],
+ current_block_parameter_keys=["create_resource_output"],
+ output_dependency_failure_class="missing_prior_block_output",
+ repair_instruction=(
+ "repair the missing prior block output dependency by binding to an actual available prior output key "
+ "or changing the producing/current code block so the dependency is real; do not invent a workflow "
+ "parameter for this missing output key."
+ ),
+ )
+ ctx = _ctx(
+ block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER,
+ last_code_authoring_repair_context=repair_context,
+ )
+
+ prompt = agent_module._code_authoring_repair_context_prompt(ctx)
+
+ assert "reason_code: runtime_missing_output_dependency" in prompt
+ assert "missing_output_key: create_resource_output" in prompt
+ assert "available_output_keys: search_output" in prompt
+ assert "current_block_parameter_keys: create_resource_output" in prompt
+ assert "bind to an actual available_output_key" in prompt
+ assert "do not create a workflow parameter for missing_output_key" in prompt
+ assert "create workflow string parameter key create_resource_output" not in prompt
+
def test_synthesized_parameter_binding_prompt_uses_exact_key(self) -> None:
repair_context = CodeAuthoringRepairContext(
block_label="order_status",
diff --git a/tests/unit/test_copilot_build_test_outcome.py b/tests/unit/test_copilot_build_test_outcome.py
index a95a2c660..c53c9930e 100644
--- a/tests/unit/test_copilot_build_test_outcome.py
+++ b/tests/unit/test_copilot_build_test_outcome.py
@@ -392,6 +392,33 @@ def test_authoring_repair_context_produces_structural_recorded_outcome() -> None
)
+def test_authoring_repair_context_missing_output_fields_affect_structural_outcome() -> None:
+ context = CodeAuthoringRepairContext(
+ block_label="read_resource_table",
+ reason_code="runtime_missing_output_dependency",
+ missing_output_key="create_resource_output",
+ available_output_keys=["search_output"],
+ current_block_parameter_keys=["create_resource_output"],
+ output_dependency_failure_class="missing_prior_block_output",
+ )
+
+ outcome = recorded_outcome_from_authoring_repair_context(context)
+
+ assert outcome.reason_code == "runtime_missing_output_dependency"
+ assert (
+ outcome.structural_key
+ != recorded_outcome_from_authoring_repair_context(
+ context.model_copy(update={"missing_output_key": "verify_resource_output"})
+ ).structural_key
+ )
+ assert (
+ outcome.structural_key
+ != recorded_outcome_from_authoring_repair_context(
+ context.model_copy(update={"available_output_keys": ["search_output", "verify_resource_output"]})
+ ).structural_key
+ )
+
+
def test_author_time_reject_structural_payloads_make_distinct_keys() -> None:
first = recorded_outcome_from_author_time_reject(
reason_code="schema_incompatibility",
diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.py
index b0939e0d2..02b2eaa60 100644
--- a/tests/unit/test_copilot_code_block_persist_seam.py
+++ b/tests/unit/test_copilot_code_block_persist_seam.py
@@ -37,6 +37,7 @@ from skyvern.forge.sdk.copilot.enforcement import (
MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS,
_check_enforcement,
)
+from skyvern.forge.sdk.copilot.output_contracts import code_block_available_binding_keys_by_label
from skyvern.forge.sdk.copilot.output_utils import sanitize_tool_result_for_llm
from skyvern.forge.sdk.copilot.reached_download_target import ReachedDownloadTarget
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
@@ -2088,6 +2089,42 @@ class TestCodeBlockParameterPersistSeam:
== "Workflow YAML nesting is too deep to validate."
)
+ def test_output_contract_graph_preserves_branch_isolation(self) -> None:
+ workflow_yaml = _yaml(
+ """
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: start_search
+ code: |
+ return {"ok": True}
+ - block_type: task
+ label: choose_path
+ branch_conditions:
+ - blocks:
+ - block_type: code
+ label: branch_a
+ code: |
+ return start_search_output
+ - blocks:
+ - block_type: code
+ label: branch_b
+ code: |
+ return start_search_output
+ - block_type: code
+ label: after_choice
+ code: |
+ return choose_path_output
+ """
+ )
+
+ available_by_label = code_block_available_binding_keys_by_label(workflow_yaml)
+
+ assert available_by_label["branch_a"] == ["start_search_output"]
+ assert available_by_label["branch_b"] == ["start_search_output"]
+ assert "branch_a_output" not in available_by_label["branch_b"]
+ assert available_by_label["after_choice"] == ["choose_path_output", "start_search_output"]
+
@pytest.mark.asyncio
async def test_declared_workflow_string_parameter_key_is_accepted(self, monkeypatch: pytest.MonkeyPatch) -> None:
_stub_successful_update(monkeypatch)
diff --git a/tests/unit/test_copilot_diagnosis_repair_contract.py b/tests/unit/test_copilot_diagnosis_repair_contract.py
index 409a513dc..7580f4e1a 100644
--- a/tests/unit/test_copilot_diagnosis_repair_contract.py
+++ b/tests/unit/test_copilot_diagnosis_repair_contract.py
@@ -28,6 +28,7 @@ from skyvern.forge.sdk.copilot.runtime_authoring_repair import (
finalize_runtime_authoring_repair_context_from_page_observation,
inject_runtime_authoring_repair_context,
post_run_inspection_cleanly_matches,
+ record_pending_runtime_authoring_repair_context,
)
from skyvern.forge.sdk.copilot.tools import run_execution as run_execution_module
from skyvern.forge.sdk.copilot.tools.composition_capture import store_post_run_page_evidence
@@ -51,6 +52,72 @@ def _ctx() -> CopilotContext:
)
+def _runtime_output_dependency_yaml(*, available: bool = False) -> str:
+ producer_label = "create_or_verify_resource" if available else "create_resource"
+ return f"""
+workflow_definition:
+ blocks:
+ - block_type: code
+ label: {producer_label}
+ code: |
+ return {{"ok": True}}
+ - block_type: code
+ label: read_resource_table
+ parameter_keys: [create_or_verify_resource_output]
+ code: |
+ resource = create_or_verify_resource_output["id"]
+ return {{"resource": resource}}
+"""
+
+
+def _runtime_declared_output_named_input_yaml() -> str:
+ return """
+workflow_definition:
+ parameters:
+ - key: create_or_verify_resource_output
+ parameter_type: workflow
+ workflow_parameter_type: string
+ blocks:
+ - block_type: code
+ label: read_resource_table
+ parameter_keys: [create_or_verify_resource_output]
+ code: |
+ resource = create_or_verify_resource_output["id"]
+ return {"resource": resource}
+"""
+
+
+def _runtime_declared_non_string_output_named_input_yaml() -> str:
+ return """
+workflow_definition:
+ parameters:
+ - key: create_or_verify_resource_output
+ parameter_type: workflow
+ workflow_parameter_type: number
+ blocks:
+ - block_type: code
+ label: read_resource_table
+ parameter_keys: [create_or_verify_resource_output]
+ code: |
+ resource = create_or_verify_resource_output["id"]
+ return {"resource": resource}
+"""
+
+
+def _runtime_output_substring_only_yaml() -> str:
+ return """
+workflow_definition:
+ blocks:
+ - block_type: code
+ label: read_resource_table
+ code: |
+ # foo_output appears in prose only.
+ data = {"foo_output": {"id": "fixture"}}
+ literal = "foo_output"
+ return data["foo_output"]
+"""
+
+
def _satisfied_completion_verification() -> CompletionVerificationResult:
return CompletionVerificationResult(
status="evaluated",
@@ -245,6 +312,45 @@ def test_missing_required_output_key_repair_identity_uses_structural_context_onl
assert contract.diagnosis_result.root_cause_identity.error_class == "code_authoring_missing_required_output_key"
+def test_runtime_missing_output_dependency_identity_uses_key_and_available_contracts() -> None:
+ base = CodeAuthoringRepairContext(
+ block_label="read_resource_table",
+ reason_code="runtime_missing_output_dependency",
+ missing_output_key="create_resource_output",
+ available_output_keys=["search_output", "verify_output"],
+ current_block_parameter_keys=["create_resource_output"],
+ output_dependency_failure_class="missing_prior_block_output",
+ )
+ reordered = base.model_copy(update={"available_output_keys": ["verify_output", "search_output"]})
+ different_key = base.model_copy(update={"missing_output_key": "verify_resource_output"})
+ different_available = base.model_copy(update={"available_output_keys": ["search_output"]})
+
+ base_signature = build_diagnosis_repair_contract(
+ source_tool="update_and_run_blocks",
+ result=_authoring_repair_result(base),
+ ctx=_ctx(),
+ ).to_trace_data()["root_cause_signature"]
+ reordered_signature = build_diagnosis_repair_contract(
+ source_tool="update_and_run_blocks",
+ result=_authoring_repair_result(reordered),
+ ctx=_ctx(),
+ ).to_trace_data()["root_cause_signature"]
+ different_key_signature = build_diagnosis_repair_contract(
+ source_tool="update_and_run_blocks",
+ result=_authoring_repair_result(different_key),
+ ctx=_ctx(),
+ ).to_trace_data()["root_cause_signature"]
+ different_available_signature = build_diagnosis_repair_contract(
+ source_tool="update_and_run_blocks",
+ result=_authoring_repair_result(different_available),
+ ctx=_ctx(),
+ ).to_trace_data()["root_cause_signature"]
+
+ assert base_signature == reordered_signature
+ assert different_key_signature != base_signature
+ assert different_available_signature != base_signature
+
+
def test_runtime_authoring_repair_context_identity_includes_bounded_page_state() -> None:
base = CodeAuthoringRepairContext(
block_label="search_registry",
@@ -431,6 +537,153 @@ def test_failed_run_injects_pending_runtime_authoring_context_before_page_observ
assert contract.repair_decision.target_blocks == ["search_registry"]
+def test_runtime_key_error_for_missing_prior_output_records_typed_authoring_context() -> None:
+ ctx = _ctx()
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ ctx.workflow_yaml = _runtime_output_dependency_yaml(available=False)
+ result = {
+ "ok": False,
+ "error": "Run failed.",
+ "data": {
+ "workflow_run_id": "wr_missing_output",
+ "overall_status": "failed",
+ "blocks": [
+ {
+ "label": "read_resource_table",
+ "status": "failed",
+ "failure_reason": "KeyError: 'create_or_verify_resource_output'",
+ }
+ ],
+ },
+ }
+
+ record_pending_runtime_authoring_repair_context(ctx, result)
+ inject_runtime_authoring_repair_context(ctx, result)
+
+ repair_context = ctx.last_code_authoring_repair_context
+ assert isinstance(repair_context, CodeAuthoringRepairContext)
+ assert repair_context.reason_code == "runtime_missing_output_dependency"
+ assert repair_context.block_label == "read_resource_table"
+ assert repair_context.workflow_run_id == "wr_missing_output"
+ assert repair_context.output_dependency_failure_class == "missing_prior_block_output"
+ assert repair_context.missing_output_key == "create_or_verify_resource_output"
+ assert repair_context.available_output_keys == ["create_resource_output"]
+ assert repair_context.current_block_parameter_keys == ["create_or_verify_resource_output"]
+ assert result["data"]["authoring_repair_context"] == repair_context.model_dump(mode="json")
+
+
+def test_runtime_key_error_for_available_prior_output_keeps_generic_runtime_repair() -> None:
+ ctx = _ctx()
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ ctx.workflow_yaml = _runtime_output_dependency_yaml(available=True)
+ result = {
+ "ok": False,
+ "error": "Run failed.",
+ "data": {
+ "workflow_run_id": "wr_available_output",
+ "overall_status": "failed",
+ "blocks": [
+ {
+ "label": "read_resource_table",
+ "status": "failed",
+ "failure_reason": "KeyError: 'create_or_verify_resource_output'",
+ }
+ ],
+ },
+ }
+
+ record_pending_runtime_authoring_repair_context(ctx, result)
+
+ pending_context = ctx.pending_code_authoring_runtime_repair_context
+ assert isinstance(pending_context, CodeAuthoringRepairContext)
+ assert pending_context.reason_code == "runtime_block_failure"
+ assert pending_context.missing_output_key is None
+
+
+def test_runtime_key_error_for_declared_workflow_input_keeps_generic_runtime_repair() -> None:
+ ctx = _ctx()
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ ctx.workflow_yaml = _runtime_declared_output_named_input_yaml()
+ result = {
+ "ok": False,
+ "error": "Run failed.",
+ "data": {
+ "workflow_run_id": "wr_declared_input",
+ "overall_status": "failed",
+ "blocks": [
+ {
+ "label": "read_resource_table",
+ "status": "failed",
+ "failure_reason": "KeyError: 'create_or_verify_resource_output'",
+ }
+ ],
+ },
+ }
+
+ record_pending_runtime_authoring_repair_context(ctx, result)
+
+ pending_context = ctx.pending_code_authoring_runtime_repair_context
+ assert isinstance(pending_context, CodeAuthoringRepairContext)
+ assert pending_context.reason_code == "runtime_block_failure"
+ assert pending_context.missing_output_key is None
+
+
+def test_runtime_key_error_for_declared_non_string_workflow_input_keeps_generic_runtime_repair() -> None:
+ ctx = _ctx()
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ ctx.workflow_yaml = _runtime_declared_non_string_output_named_input_yaml()
+ result = {
+ "ok": False,
+ "error": "Run failed.",
+ "data": {
+ "workflow_run_id": "wr_declared_number_input",
+ "overall_status": "failed",
+ "blocks": [
+ {
+ "label": "read_resource_table",
+ "status": "failed",
+ "failure_reason": "KeyError: 'create_or_verify_resource_output'",
+ }
+ ],
+ },
+ }
+
+ record_pending_runtime_authoring_repair_context(ctx, result)
+
+ pending_context = ctx.pending_code_authoring_runtime_repair_context
+ assert isinstance(pending_context, CodeAuthoringRepairContext)
+ assert pending_context.reason_code == "runtime_block_failure"
+ assert pending_context.missing_output_key is None
+
+
+def test_runtime_key_error_for_code_substring_only_keeps_generic_runtime_repair() -> None:
+ ctx = _ctx()
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ ctx.workflow_yaml = _runtime_output_substring_only_yaml()
+ result = {
+ "ok": False,
+ "error": "Run failed.",
+ "data": {
+ "workflow_run_id": "wr_substring_only",
+ "overall_status": "failed",
+ "blocks": [
+ {
+ "label": "read_resource_table",
+ "status": "failed",
+ "failure_reason": "KeyError: 'foo_output'",
+ }
+ ],
+ },
+ }
+
+ record_pending_runtime_authoring_repair_context(ctx, result)
+
+ pending_context = ctx.pending_code_authoring_runtime_repair_context
+ assert isinstance(pending_context, CodeAuthoringRepairContext)
+ assert pending_context.reason_code == "runtime_block_failure"
+ assert pending_context.missing_output_key is None
+
+
def _injected_repair_log(events: list[dict[str, object]]) -> dict[str, object]:
matches = [event for event in events if event.get("event") == "Injected runtime authoring repair context"]
assert len(matches) == 1