Skip to content

Automate browser based workflows with AI

SKY-11497 Fix Copilot runtime repair context routing (#6858)

Andrew Neilson3de50e3d3e4d
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 5e691a6f2..9485bff2a 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -623,6 +623,42 @@ def _code_authoring_repair_context_prompt(ctx: CopilotContext | None) -> str:
lines.append(f"source_url: {_clean_authoring_repair_prompt_atom(repair_context.source_url)}")
if repair_context.refiner_selector:
lines.append(f"refiner_selector: {_clean_authoring_repair_prompt_atom(repair_context.refiner_selector)}")
+ if repair_context.reason_code == "runtime_block_failure":
+ if repair_context.runtime_failure_reason:
+ lines.append(
+ f"runtime_failure_reason: {_clean_authoring_repair_prompt_atom(repair_context.runtime_failure_reason)}"
+ )
+ if repair_context.runtime_failure_class:
+ lines.append(
+ f"runtime_failure_class: {_clean_authoring_repair_prompt_atom(repair_context.runtime_failure_class)}"
+ )
+ if repair_context.failed_block_status:
+ lines.append(
+ f"failed_block_status: {_clean_authoring_repair_prompt_atom(repair_context.failed_block_status)}"
+ )
+ if repair_context.workflow_run_id:
+ workflow_run_id = _clean_authoring_repair_prompt_atom(repair_context.workflow_run_id)
+ if workflow_run_id:
+ lines.append(f"workflow_run_id: {workflow_run_id}")
+ if repair_context.current_origin:
+ lines.append(f"current_origin: {_clean_authoring_repair_prompt_atom(repair_context.current_origin)}")
+ lines.append(f"current_url_present: {str(repair_context.current_url_present).lower()}")
+ lines.append(f"current_title_present: {str(repair_context.current_title_present).lower()}")
+ if repair_context.page_evidence_source:
+ page_evidence_source = _clean_authoring_repair_prompt_atom(repair_context.page_evidence_source)
+ if page_evidence_source:
+ lines.append(f"page_evidence_source: {page_evidence_source}")
+ lines.append(f"observed_after_workflow_run: {str(repair_context.observed_after_workflow_run).lower()}")
+ if repair_context.page_form_summaries:
+ lines.append(f"page_forms: {_render_authoring_repair_prompt_list(repair_context.page_form_summaries)}")
+ if repair_context.page_result_summaries:
+ lines.append(f"page_results: {_render_authoring_repair_prompt_list(repair_context.page_result_summaries)}")
+ if repair_context.page_action_summaries:
+ lines.append(f"page_actions: {_render_authoring_repair_prompt_list(repair_context.page_action_summaries)}")
+ if repair_context.page_challenge_summaries:
+ lines.append(
+ f"page_challenges: {_render_authoring_repair_prompt_list(repair_context.page_challenge_summaries)}"
+ )
selector_alternative_lines = _render_selector_repair_alternatives(repair_context.selector_alternatives)
if selector_alternative_lines:
lines.append("same_page_selector_alternatives:")
@@ -646,12 +682,29 @@ def _code_authoring_repair_context_prompt(ctx: CopilotContext | None) -> str:
"list the exact key in the code block's parameter_keys, reference the exact key as a bare Python "
"variable in code, do not hardcode the eval value, and rerun via update_and_run_blocks."
)
+ if repair_context.reason_code == "synthesized_parameter_binding_ambiguous":
+ binding_action_lines = _render_unresolved_name_binding_actions(
+ repair_context.unresolved_names, available_parameter_keys
+ )
+ if binding_action_lines:
+ lines.append("binding_actions:")
+ lines.extend(binding_action_lines)
+ lines.append(
+ "For synthesized parameter binding, declare and use the exact workflow input key, include that exact "
+ "key in the code block's parameter_keys, reference it as a bare Python variable in code, do not guess "
+ "or hardcode the runtime value, and rerun via update_and_run_blocks."
+ )
if repair_context.reason_code == "ambiguous_bare_selector":
lines.append(
"For ambiguous selectors, do not re-emit the bare selector or a positional nth selector. "
"Use the same-page alternatives when they are stable, or re-scout the same page and choose a "
"stable role/name/data attribute."
)
+ if repair_context.reason_code == "runtime_block_failure":
+ lines.append(
+ "For runtime failures, adapt the next code block to the observed page state and do not re-emit "
+ "the same failing selector or name path."
+ )
lines.append(_clean_authoring_repair_prompt_atom(repair_context.repair_instruction, max_chars=260))
return "\n\n" + "\n".join(line for line in lines if line)
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index 9f18f2fa2..d6fefb547 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -147,6 +147,19 @@ class CodeAuthoringRepairContext(BaseModel):
selector_alternatives: list[dict[str, str]] = Field(default_factory=list)
allowed_global_names: list[str] = Field(default_factory=list)
allowed_helper_surface: dict[str, list[str]] = Field(default_factory=dict)
+ runtime_failure_reason: str | None = None
+ runtime_failure_class: str | None = None
+ failed_block_status: str | None = None
+ workflow_run_id: str | None = None
+ current_origin: str | None = None
+ current_url_present: bool = False
+ current_title_present: bool = False
+ page_evidence_source: str | None = None
+ observed_after_workflow_run: bool = False
+ page_form_summaries: list[str] = Field(default_factory=list)
+ page_result_summaries: list[str] = Field(default_factory=list)
+ page_action_summaries: list[str] = Field(default_factory=list)
+ page_challenge_summaries: list[str] = Field(default_factory=list)
repair_instruction: str = "add workflow-input-like names to parameter_keys, or stop referencing them."
@@ -499,6 +512,7 @@ class CopilotContext(AgentContext):
# True when the most-recent such rejection deferred to the credential-scout
# gate, so the churn backstop yields to that message instead of pre-empting it.
last_code_authoring_reject_was_credential_priority: bool = False
+ pending_code_authoring_runtime_repair_context: CodeAuthoringRepairContext | None = None
last_code_authoring_repair_context: CodeAuthoringRepairContext | None = None
# Turn-scoped monotonic marks of verified forward progress: the union of
# completion criteria the judge confirmed satisfied so far this turn, and the
diff --git a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
index 6bf184051..e6f91b220 100644
--- a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
+++ b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
@@ -350,15 +350,31 @@ def _repair_context_root_cause_identity(
payload["selector"] = selector
payload["refiner_selector"] = refiner_selector
selector_kind = "selector" if selector else ""
+ elif reason_code == "runtime_block_failure":
+ runtime_failure_class = _safe_text(repair_context.runtime_failure_class, 80)
+ payload["runtime_failure_class"] = runtime_failure_class
+ payload["failed_block_status"] = _safe_text(repair_context.failed_block_status, 80)
+ payload["current_origin"] = _safe_text(repair_context.current_origin, 120)
+ payload["current_url_present"] = repair_context.current_url_present
+ payload["current_title_present"] = repair_context.current_title_present
+ payload["page_evidence_source"] = _safe_text(repair_context.page_evidence_source, 80)
+ payload["observed_after_workflow_run"] = repair_context.observed_after_workflow_run
+ payload["page_form_summaries"] = _safe_identity_list(repair_context.page_form_summaries)
+ payload["page_result_summaries"] = _safe_identity_list(repair_context.page_result_summaries)
+ payload["page_action_summaries"] = _safe_identity_list(repair_context.page_action_summaries)
+ payload["page_challenge_summaries"] = _safe_identity_list(repair_context.page_challenge_summaries)
elif repair_context.unresolved_names:
payload["unresolved_names"] = _safe_identity_list(repair_context.unresolved_names)
+ error_class_suffix = _identity_token(reason_code)
+ if reason_code == "runtime_block_failure" and repair_context.runtime_failure_class:
+ error_class_suffix = f"{error_class_suffix}_{_identity_token(repair_context.runtime_failure_class)}"
serialized = json.dumps(payload, sort_keys=True, separators=(",", ":"))
return RepairRootCauseIdentity(
root_cause_signature=hashlib.sha256(serialized.encode("utf-8")).hexdigest(),
primary_category=_AUTHORING_REPAIR_CATEGORY,
failure_categories=(_AUTHORING_REPAIR_CATEGORY,),
- error_class=f"code_authoring_{_identity_token(reason_code)}",
+ error_class=f"code_authoring_{error_class_suffix}",
selector_kind=selector_kind,
selector=selector,
)
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index 799450bc5..e4d018d21 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -204,6 +204,7 @@ class AgentContext:
repeated_failure_nudge_emitted_at_streak: int = 0
code_authoring_guardrail_reject_count: int = 0
last_code_authoring_reject_was_credential_priority: bool = False
+ pending_code_authoring_runtime_repair_context: CodeAuthoringRepairContext | None = None
last_code_authoring_repair_context: CodeAuthoringRepairContext | None = None
challenge_gated_proxy_retry_count: int = 0
last_test_non_retriable_nav_error: str | None = None
diff --git a/skyvern/forge/sdk/copilot/runtime_authoring_repair.py b/skyvern/forge/sdk/copilot/runtime_authoring_repair.py
new file mode 100644
index 000000000..d73fea69f
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/runtime_authoring_repair.py
@@ -0,0 +1,318 @@
+from __future__ import annotations
+
+import re
+from typing import Any, TypeGuard
+
+from skyvern.forge.sdk.copilot.composition_evidence import has_bounded_page_schema
+from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, normalize_block_authoring_policy
+from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext
+from skyvern.forge.sdk.copilot.failure_tracking import ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY
+from skyvern.forge.sdk.copilot.request_policy import redact_raw_secrets_for_prompt
+from skyvern.forge.sdk.copilot.run_outcome import trusted_terminal_challenge_category_name
+from skyvern.forge.sdk.copilot.workflow_credential_utils import url_origin
+
+_RUNTIME_AUTHORING_REASON_CODE = "runtime_block_failure"
+_RUNTIME_SUMMARY_MAX_CHARS = 120
+_RUNTIME_SUMMARY_MAX_ITEMS = 5
+_INSPECT_PAGE_SOURCE_TOOL = "inspect_page_for_composition"
+
+
+def is_runtime_authoring_repair_context(repair_context: object) -> TypeGuard[CodeAuthoringRepairContext]:
+ return (
+ isinstance(repair_context, CodeAuthoringRepairContext)
+ and repair_context.reason_code == _RUNTIME_AUTHORING_REASON_CODE
+ )
+
+
+def clear_runtime_authoring_repair_context(copilot_ctx: Any) -> None:
+ copilot_ctx.pending_code_authoring_runtime_repair_context = None
+ if is_runtime_authoring_repair_context(getattr(copilot_ctx, "last_code_authoring_repair_context", None)):
+ copilot_ctx.last_code_authoring_repair_context = None
+
+
+def _bounded_runtime_text(value: Any, max_chars: int = _RUNTIME_SUMMARY_MAX_CHARS) -> str:
+ if not isinstance(value, str):
+ return ""
+ text = redact_raw_secrets_for_prompt(" ".join(value.split()))
+ return text[:max_chars]
+
+
+def _runtime_failure_class(reason: str) -> str:
+ reason_lower = reason.lower()
+ if "timeout" in reason_lower and any(token in reason_lower for token in ("locator", "selector", "element")):
+ return "timeout_waiting_for_selector"
+ if "not found" in reason_lower and any(token in reason_lower for token in ("locator", "selector", "element")):
+ return "selector_not_found"
+ normalized = re.sub(r"[^a-z0-9]+", "_", reason_lower).strip("_")
+ return normalized[:80].strip("_") or "runtime_failure"
+
+
+def _origin_from_runtime_url(value: Any) -> str | None:
+ if not isinstance(value, str) or not value.strip():
+ return None
+ return url_origin(value)
+
+
+def _runtime_summary_entry(entry: Any, keys: tuple[str, ...]) -> str:
+ if not isinstance(entry, dict):
+ return _bounded_runtime_text(entry)
+ parts = [
+ _bounded_runtime_text(entry.get(key), 60)
+ if not isinstance(entry.get(key), bool)
+ else ("disabled" if entry.get(key) is True else "enabled")
+ for key in keys
+ ]
+ return _bounded_runtime_text(" ".join(part for part in parts if part))
+
+
+def _runtime_summary_list(value: Any, keys: tuple[str, ...]) -> list[str]:
+ if not isinstance(value, list):
+ return []
+ summaries: list[str] = []
+ for entry in value[:_RUNTIME_SUMMARY_MAX_ITEMS]:
+ summary = _runtime_summary_entry(entry, keys)
+ if summary:
+ summaries.append(summary)
+ return summaries
+
+
+def _post_run_terminal_page_evidence(evidence: dict[str, Any]) -> bool:
+ if evidence.get("observed_after_workflow_run") is not True:
+ return False
+ challenge_state = evidence.get("challenge_state")
+ if isinstance(challenge_state, dict):
+ if challenge_state.get("gates_submit_controls") is True:
+ return True
+ if (
+ challenge_state.get("detected") is True
+ and challenge_state.get("requires_human_verification") is True
+ and _runtime_summary_list(evidence.get("forms"), ("label", "selector"))
+ ):
+ return True
+ indicators = evidence.get("anti_bot_indicators")
+ has_indicators = isinstance(indicators, list) and any(isinstance(item, str) and item.strip() for item in indicators)
+ controls = evidence.get("challenge_controls")
+ has_controls = isinstance(controls, list) and any(isinstance(item, dict) for item in controls)
+ return has_indicators and has_controls
+
+
+def _first_runtime_failed_block(data: dict[str, Any]) -> dict[str, Any] | None:
+ blocks = data.get("blocks")
+ if not isinstance(blocks, list):
+ return None
+ for block in blocks:
+ if not isinstance(block, dict):
+ continue
+ status = str(block.get("status") or "").lower()
+ if status in {"failed", "terminated", "canceled", "timed_out"}:
+ return block
+ return None
+
+
+def record_pending_runtime_authoring_repair_context(copilot_ctx: Any, result: dict[str, Any]) -> None:
+ if bool(result.get("ok", False)):
+ clear_runtime_authoring_repair_context(copilot_ctx)
+ return
+ data = result.get("data")
+ if not isinstance(data, dict):
+ clear_runtime_authoring_repair_context(copilot_ctx)
+ return
+ run_id = data.get("workflow_run_id")
+ if not isinstance(run_id, str) or not run_id:
+ clear_runtime_authoring_repair_context(copilot_ctx)
+ return
+ block = _first_runtime_failed_block(data)
+ failure_reason = ""
+ block_label = _bounded_runtime_text(data.get("frontier_start_label"), 80)
+ failed_block_status = _bounded_runtime_text(data.get("overall_status"), 40)
+ if block is not None:
+ block_label = _bounded_runtime_text(block.get("label"), 80) or block_label
+ failed_block_status = _bounded_runtime_text(block.get("status"), 40) or failed_block_status
+ failure_reason = _bounded_runtime_text(block.get("failure_reason"), 240)
+ failure_reason = failure_reason or _bounded_runtime_text(data.get("failure_reason"), 240)
+ failure_reason = failure_reason or _bounded_runtime_text(result.get("error"), 240)
+ if not block_label or not failure_reason:
+ clear_runtime_authoring_repair_context(copilot_ctx)
+ return
+ if is_runtime_authoring_repair_context(getattr(copilot_ctx, "last_code_authoring_repair_context", None)):
+ copilot_ctx.last_code_authoring_repair_context = None
+ copilot_ctx.pending_code_authoring_runtime_repair_context = CodeAuthoringRepairContext(
+ block_label=block_label,
+ reason_code=_RUNTIME_AUTHORING_REASON_CODE,
+ runtime_failure_reason=failure_reason,
+ runtime_failure_class=_runtime_failure_class(failure_reason),
+ failed_block_status=failed_block_status or None,
+ workflow_run_id=run_id,
+ repair_instruction=(
+ "adapt the next code block to the observed page state and do not re-emit the same failing selector "
+ "or name path."
+ ),
+ )
+
+
+def _authority_requires_ask(copilot_ctx: Any) -> bool:
+ authority = getattr(getattr(copilot_ctx, "turn_intent", None), "authority", None)
+ return getattr(authority, "requires_user_input", False) or getattr(authority, "may_update_workflow", True) is False
+
+
+def _policy_allows_runtime_authoring_repair(copilot_ctx: Any) -> bool:
+ return normalize_block_authoring_policy(getattr(copilot_ctx, "block_authoring_policy", None)) == (
+ BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ )
+
+
+def _error_text_requires_stop(copilot_ctx: Any, data: dict[str, Any], result: dict[str, Any] | None = None) -> bool:
+ if getattr(copilot_ctx, "last_test_non_retriable_nav_error", None):
+ return True
+ text_values = [data.get("failure_reason"), data.get("skip_reason")]
+ if result is not None:
+ text_values.append(result.get("error"))
+ text = " ".join(str(value).lower() for value in text_values if value)
+ return (
+ "browser session not found" in text
+ or "no browser context" in text
+ or ("session not found" in text and "browser" in text)
+ or ("404" in text and "browser session" in text)
+ )
+
+
+def _error_text_requires_ask(data: dict[str, Any], result: dict[str, Any] | None = None) -> bool:
+ text_values = [data.get("failure_reason"), data.get("skip_reason"), data.get("failure_type")]
+ if result is not None:
+ text_values.append(result.get("error"))
+ text = " ".join(str(value).lower() for value in text_values if value)
+ return (
+ "workflow_credential_inputs_unbound" in text
+ or "credential inputs unbound" in text
+ or "required credentials are not configured" in text
+ or "missing_credential_or_init" in text
+ )
+
+
+def _pending_state_has_stop_or_ask_precedence(copilot_ctx: Any, pending: CodeAuthoringRepairContext) -> bool:
+ data = {
+ "failure_reason": pending.runtime_failure_reason,
+ "skip_reason": pending.runtime_failure_reason,
+ "failure_type": pending.runtime_failure_class,
+ }
+ if _authority_requires_ask(copilot_ctx):
+ return True
+ return _error_text_requires_stop(copilot_ctx, data) or _error_text_requires_ask(data)
+
+
+def _result_has_terminal_or_ask_precedence(copilot_ctx: Any, data: dict[str, Any], result: dict[str, Any]) -> bool:
+ if _authority_requires_ask(copilot_ctx):
+ return True
+ if _error_text_requires_stop(copilot_ctx, data, result):
+ return True
+ if _error_text_requires_ask(data, result):
+ return True
+ if data.get("active_run_terminal_evidence_detected") is True:
+ return True
+ if data.get("skip_reason") == "workflow_credential_inputs_unbound":
+ return True
+ if data.get("failure_type") in {"schema_incompatibility", "missing_credential_or_init"}:
+ return True
+ categories = data.get("failure_categories")
+ if not isinstance(categories, list):
+ return False
+ for entry in categories:
+ if not isinstance(entry, dict):
+ continue
+ category = entry.get("category")
+ if category in {
+ "UNRECOVERABLE_TOOL_ERROR",
+ ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,
+ "ANTI_BOT_DETECTION",
+ }:
+ return True
+ if trusted_terminal_challenge_category_name(entry):
+ return True
+ return False
+
+
+def _matching_bounded_post_run_inspection(
+ copilot_ctx: Any, pending: CodeAuthoringRepairContext
+) -> dict[str, Any] | None:
+ evidence = getattr(copilot_ctx, "composition_page_evidence", None)
+ if not isinstance(evidence, dict):
+ return None
+ if evidence.get("source_tool") != _INSPECT_PAGE_SOURCE_TOOL:
+ clear_runtime_authoring_repair_context(copilot_ctx)
+ return None
+ if evidence.get("observed_after_workflow_run") is not True:
+ clear_runtime_authoring_repair_context(copilot_ctx)
+ return None
+ run_id = evidence.get("workflow_run_id")
+ if not isinstance(run_id, str) or run_id != pending.workflow_run_id:
+ clear_runtime_authoring_repair_context(copilot_ctx)
+ return None
+ if not has_bounded_page_schema(evidence):
+ clear_runtime_authoring_repair_context(copilot_ctx)
+ return None
+ if _post_run_terminal_page_evidence(evidence):
+ clear_runtime_authoring_repair_context(copilot_ctx)
+ return None
+ return evidence
+
+
+def finalize_runtime_authoring_repair_context_from_page_observation(
+ copilot_ctx: Any,
+) -> CodeAuthoringRepairContext | None:
+ pending = getattr(copilot_ctx, "pending_code_authoring_runtime_repair_context", None)
+ if not is_runtime_authoring_repair_context(pending):
+ return None
+ if not _policy_allows_runtime_authoring_repair(copilot_ctx) or _pending_state_has_stop_or_ask_precedence(
+ copilot_ctx, pending
+ ):
+ clear_runtime_authoring_repair_context(copilot_ctx)
+ return None
+ evidence = _matching_bounded_post_run_inspection(copilot_ctx, pending)
+ if evidence is None:
+ return None
+ current_url = evidence.get("current_url") or evidence.get("inspected_url")
+ page_title = evidence.get("page_title") or evidence.get("title")
+ finalized = pending.model_copy(
+ update={
+ "current_origin": _origin_from_runtime_url(current_url),
+ "current_url_present": isinstance(current_url, str) and bool(current_url.strip()),
+ "current_title_present": isinstance(page_title, str) and bool(page_title.strip()),
+ "page_evidence_source": _bounded_runtime_text(evidence.get("source_tool"), 80) or None,
+ "observed_after_workflow_run": True,
+ "page_form_summaries": _runtime_summary_list(evidence.get("forms"), ("label", "selector")),
+ "page_result_summaries": _runtime_summary_list(evidence.get("result_containers"), ("label", "text")),
+ "page_action_summaries": _runtime_summary_list(
+ evidence.get("navigation_targets"), ("label", "selector", "disabled")
+ ),
+ "page_challenge_summaries": _runtime_summary_list(
+ evidence.get("challenge_controls"), ("text", "selector", "disabled")
+ ),
+ }
+ )
+ copilot_ctx.last_code_authoring_repair_context = finalized
+ copilot_ctx.pending_code_authoring_runtime_repair_context = None
+ return finalized
+
+
+def inject_runtime_authoring_repair_context(copilot_ctx: Any, result: dict[str, Any]) -> None:
+ data = result.get("data")
+ if not isinstance(data, dict):
+ return
+ if _result_has_terminal_or_ask_precedence(copilot_ctx, data, result):
+ clear_runtime_authoring_repair_context(copilot_ctx)
+ data.pop("authoring_repair_context", None)
+ return
+ repair_context = finalize_runtime_authoring_repair_context_from_page_observation(copilot_ctx)
+ if repair_context is None:
+ pending = getattr(copilot_ctx, "pending_code_authoring_runtime_repair_context", None)
+ if not is_runtime_authoring_repair_context(pending):
+ return
+ if not _policy_allows_runtime_authoring_repair(copilot_ctx) or _pending_state_has_stop_or_ask_precedence(
+ copilot_ctx, pending
+ ):
+ clear_runtime_authoring_repair_context(copilot_ctx)
+ data.pop("authoring_repair_context", None)
+ return
+ repair_context = pending
+ copilot_ctx.last_code_authoring_repair_context = repair_context
+ data["authoring_repair_context"] = repair_context.model_dump(mode="json")
diff --git a/skyvern/forge/sdk/copilot/tools/composition_capture.py b/skyvern/forge/sdk/copilot/tools/composition_capture.py
index ed0cdad30..eb1459aeb 100644
--- a/skyvern/forge/sdk/copilot/tools/composition_capture.py
+++ b/skyvern/forge/sdk/copilot/tools/composition_capture.py
@@ -28,6 +28,9 @@ from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.failure_tracking import ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY
from skyvern.forge.sdk.copilot.llm_config import resolve_fast_copilot_handler
from skyvern.forge.sdk.copilot.loop_detection import record_tool_step_result_for_ctx
+from skyvern.forge.sdk.copilot.runtime_authoring_repair import (
+ finalize_runtime_authoring_repair_context_from_page_observation,
+)
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
from ._shared import (
@@ -840,6 +843,7 @@ async def _inspect_page_for_composition_impl(
if bypass_budget_for_post_run_current_page:
copilot_ctx.post_run_current_page_inspection_workflow_run_id = run_id
copilot_ctx.composition_page_evidence = evidence
+ finalize_runtime_authoring_repair_context_from_page_observation(copilot_ctx)
if (
isinstance(run_id, str)
and run_id
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index c839d416b..7319d617c 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -83,6 +83,11 @@ from skyvern.forge.sdk.copilot.runtime import (
AgentContext,
ensure_browser_session,
)
+from skyvern.forge.sdk.copilot.runtime_authoring_repair import (
+ clear_runtime_authoring_repair_context,
+ inject_runtime_authoring_repair_context,
+ record_pending_runtime_authoring_repair_context,
+)
from skyvern.forge.sdk.copilot.terminal_predicates import outcome_fully_verified
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
from skyvern.forge.sdk.copilot.turn_halt import (
@@ -2003,6 +2008,7 @@ def _record_run_blocks_result(
copilot_ctx.post_run_page_observation_workflow_run_id = None
copilot_ctx.post_run_page_observation_after_failed_test = False
copilot_ctx.post_run_current_page_inspection_workflow_run_id = None
+ record_pending_runtime_authoring_repair_context(copilot_ctx, result)
structured_blocker = _run_blocks_structured_blocker_message(result, copilot_ctx)
anti_bot_match, empty_data_blocks, failure_categories = _analyze_run_blocks(result, copilot_ctx)
@@ -2071,6 +2077,7 @@ def _record_run_blocks_result(
stash_turn_halt_from_blocker_signal(copilot_ctx, signal, source="run_execution")
if terminal_challenge is not None:
+ clear_runtime_authoring_repair_context(copilot_ctx)
# A structured challenge is the more actionable terminal blocker when
# artifact-health evidence and challenge evidence appear in the same run.
blocked_verification = _terminal_challenge_completion_verification(
@@ -2424,6 +2431,7 @@ def _record_diagnosis_repair_contract(
result: dict[str, Any],
workflow_updated: bool = False,
) -> DiagnosisRepairContract:
+ inject_runtime_authoring_repair_context(copilot_ctx, result)
contract = build_diagnosis_repair_contract(
source_tool=source_tool,
result=result,
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index 2ef497508..9c57479c2 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -379,12 +379,14 @@ def _normalize_code_artifact_metadata(
*,
impose_defaults: bool = False,
scout_trajectory: list[ScoutedInteraction] | None = None,
+ verified_runtime_output_paths_by_label: Mapping[str, set[str]] | None = None,
) -> tuple[dict[str, dict[str, Any]], str | None]:
result = _normalize_code_artifact_metadata_detailed(
raw_metadata,
workflow_yaml,
impose_defaults=impose_defaults,
scout_trajectory=scout_trajectory,
+ verified_runtime_output_paths_by_label=verified_runtime_output_paths_by_label,
)
return result.normalized, result.error
@@ -395,6 +397,7 @@ def _normalize_code_artifact_metadata_detailed(
*,
impose_defaults: bool = False,
scout_trajectory: list[ScoutedInteraction] | None = None,
+ verified_runtime_output_paths_by_label: Mapping[str, set[str]] | None = None,
) -> CodeArtifactNormalization:
"""Normalize submitted artifact metadata at the persist seam.
@@ -506,7 +509,14 @@ def _normalize_code_artifact_metadata_detailed(
schema_conformance_error = _extraction_schema_conformance_error(label, dumped, block_code)
if schema_conformance_error is not None:
item_violations.append(schema_conformance_error)
- schema_incompatibility = _extraction_schema_incompatibility(label, dumped, block_code)
+ schema_incompatibility = _extraction_schema_incompatibility(
+ label,
+ dumped,
+ block_code,
+ verified_runtime_output_paths=(
+ verified_runtime_output_paths_by_label.get(label) if verified_runtime_output_paths_by_label else None
+ ),
+ )
if schema_incompatibility is not None:
schema_incompatibilities.append(schema_incompatibility)
item_violations.append(render_schema_incompatibility_agent_steer(schema_incompatibility))
@@ -2405,6 +2415,33 @@ class _SynthesizedParameterReconciliation(NamedTuple):
violations: list[str]
aliases: dict[str, str]
expressions: dict[str, str]
+ repair_context: CodeAuthoringRepairContext | None = None
+
+
+_SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_REASON_CODE = "synthesized_parameter_binding_ambiguous"
+
+
+def _synthesized_parameter_binding_repair_context(
+ *,
+ parsed: Mapping[str, Any],
+ code_block: Mapping[str, Any],
+ synthesized_key: str,
+ parameter_keys: list[str],
+) -> CodeAuthoringRepairContext:
+ available_parameter_keys = sorted(_declared_string_workflow_parameter_keys(parsed))
+ binding_candidates = [synthesized_key] + [key for key in available_parameter_keys if key != synthesized_key]
+ return CodeAuthoringRepairContext(
+ block_label=str(code_block.get("label") or ""),
+ reason_code=_SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_REASON_CODE,
+ unresolved_names=[synthesized_key],
+ parameter_keys=list(parameter_keys),
+ available_parameter_keys=available_parameter_keys,
+ binding_candidates=binding_candidates,
+ repair_instruction=(
+ f"Declare and use workflow string parameter `{synthesized_key}` exactly, include it in parameter_keys, "
+ "reference it as a bare Python variable in code, and rerun via update_and_run_blocks."
+ ),
+ )
def _reconcile_synthesized_parameters(
@@ -2437,6 +2474,7 @@ def _reconcile_synthesized_parameters(
violations: list[str] = []
aliases: dict[str, str] = {}
expressions: dict[str, str] = {}
+ repair_context: CodeAuthoringRepairContext | None = None
non_credential_synthesized = [param for param in synthesized_parameters if not param.get("credential_id")]
typed_lengths = [
int(interaction.get("typed_length") or 0)
@@ -2573,6 +2611,13 @@ def _reconcile_synthesized_parameters(
violations.append(
f"Unable to bind synthesized parameter `{key}`: missing submitted workflow parameter and literal binding is ambiguous."
)
+ if repair_context is None:
+ repair_context = _synthesized_parameter_binding_repair_context(
+ parsed=parsed,
+ code_block=code_block,
+ synthesized_key=key,
+ parameter_keys=parameter_keys,
+ )
continue
typed_length = typed_lengths[0] if len(typed_lengths) == 1 else None
@@ -2618,7 +2663,7 @@ def _reconcile_synthesized_parameters(
code_block["parameter_keys"] = parameter_keys
# Submitted synthesized parameter rows are re-derived from scout evidence before persistence.
code_block.pop("parameters", None)
- return _SynthesizedParameterReconciliation(parameter_keys, violations, aliases, expressions)
+ return _SynthesizedParameterReconciliation(parameter_keys, violations, aliases, expressions, repair_context)
def _maybe_impose_synthesized_code_block(workflow_yaml: str, ctx: AgentContext) -> _SynthesizedCodeImpositionResult:
@@ -2702,6 +2747,8 @@ def _maybe_impose_synthesized_code_block(workflow_yaml: str, ctx: AgentContext)
scout_trajectory=scout_trajectory,
)
violations.extend(parameter_reconciliation.violations)
+ if repair_context is None:
+ repair_context = parameter_reconciliation.repair_context
if violations:
return _SynthesizedCodeImpositionResult(
workflow_yaml=workflow_yaml,
@@ -3136,6 +3183,51 @@ def _top_level_path_segment(path: str) -> str:
return head.strip()
+_STRUCTURAL_RUNTIME_OUTPUT_KEY_RE = re.compile(r"^[a-z]+(?:_[a-z]+)*(?:_[0-9])?$")
+_SENSITIVE_RUNTIME_OUTPUT_KEY_TERMS = frozenset(
+ {"api_key", "access_key", "password", "secret", "token", "credential", "email"}
+)
+
+
+def _is_structural_runtime_output_key(key: str) -> bool:
+ return (
+ _STRUCTURAL_RUNTIME_OUTPUT_KEY_RE.fullmatch(key) is not None
+ and not keyword.iskeyword(key)
+ and key not in _SENSITIVE_RUNTIME_OUTPUT_KEY_TERMS
+ and not any(part in _SENSITIVE_RUNTIME_OUTPUT_KEY_TERMS for part in key.split("_"))
+ )
+
+
+def _verified_runtime_output_contract_paths(value: object, *, prefix: str = "") -> set[str]:
+ paths: set[str] = set()
+ if isinstance(value, Mapping):
+ for raw_key, child in value.items():
+ if not isinstance(raw_key, str):
+ continue
+ key = raw_key.strip()
+ if not _is_structural_runtime_output_key(key):
+ continue
+ path = f"{prefix}.{key}" if prefix else key
+ paths.add(path)
+ paths |= _verified_runtime_output_contract_paths(child, prefix=path)
+ return paths
+ if isinstance(value, list):
+ for item in value:
+ paths |= _verified_runtime_output_contract_paths(item, prefix=prefix)
+ return paths
+
+
+def _verified_runtime_output_contract_paths_by_label(ctx: AgentContext, workflow_yaml: str) -> dict[str, set[str]]:
+ if _copilot_block_authoring_policy(ctx) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
+ return {}
+ code_block_labels = set(_workflow_yaml_code_blocks_by_label(workflow_yaml))
+ return {
+ label: paths
+ for label, output in ctx.verified_block_outputs.items()
+ if label in code_block_labels and (paths := _verified_runtime_output_contract_paths(output))
+ }
+
+
def _known_output_contract_paths(artifact: Mapping[str, Any], code: str) -> set[str]:
"""Top-level field names the block is known to produce: the snippet's return-dict
keys plus the confirmed `goal_value_paths`' top-level segments. Empty when neither
@@ -3162,7 +3254,11 @@ def _schema_property_summary(schema: Mapping[str, Any]) -> str:
def _extraction_schema_incompatibility(
- label: str, artifact: Mapping[str, Any], code: str
+ label: str,
+ artifact: Mapping[str, Any],
+ code: str,
+ *,
+ verified_runtime_output_paths: set[str] | None = None,
) -> SchemaIncompatibility | None:
"""Detect an edited `extraction_schema` whose object property names overlap NONE of
the block's known output contract. Unlike `_extraction_schema_conformance_error`,
@@ -3172,6 +3268,8 @@ def _extraction_schema_incompatibility(
if _is_download_intent(artifact, code) or not code.strip():
return None
known_paths = _known_output_contract_paths(artifact, code)
+ if verified_runtime_output_paths:
+ known_paths |= {_top_level_path_segment(path) for path in verified_runtime_output_paths}
if not known_paths:
return None
incompatible: set[str] = set()
@@ -3713,6 +3811,7 @@ async def _update_workflow(
workflow_yaml,
impose_defaults=_copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER,
scout_trajectory=scout_trajectory if isinstance(scout_trajectory, list) else None,
+ verified_runtime_output_paths_by_label=_verified_runtime_output_contract_paths_by_label(ctx, workflow_yaml),
)
code_artifact_metadata = normalization.normalized
code_artifact_metadata_error = normalization.error
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index e9e0728f1..852c6fa43 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -724,6 +724,30 @@ workflow_definition:
assert "do not hardcode the eval value" in prompt
assert "rerun via update_and_run_blocks" in prompt
+ def test_synthesized_parameter_binding_prompt_uses_exact_key(self) -> None:
+ repair_context = CodeAuthoringRepairContext(
+ block_label="order_status",
+ reason_code="synthesized_parameter_binding_ambiguous",
+ unresolved_names=["enter_confirmation"],
+ parameter_keys=["enter_confirmation"],
+ available_parameter_keys=["confirmation_number"],
+ binding_candidates=["enter_confirmation", "confirmation_number"],
+ )
+ ctx = _ctx(
+ block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER,
+ last_code_authoring_repair_context=repair_context,
+ )
+
+ prompt = agent_module._code_authoring_repair_context_prompt(ctx)
+
+ assert "reason_code: synthesized_parameter_binding_ambiguous" in prompt
+ assert "binding_candidates: enter_confirmation, confirmation_number" in prompt
+ assert "declare and use the exact workflow input key" in prompt
+ assert "include that exact key in the code block's parameter_keys" in prompt
+ assert "reference it as a bare Python variable in code" in prompt
+ assert "do not guess or hardcode the runtime value" in prompt
+ assert "rerun via update_and_run_blocks" in prompt
+
def test_ambiguous_selector_repair_context_prompt_includes_selector_details(self) -> None:
repair_context = CodeAuthoringRepairContext(
block_label="order_status",
@@ -775,6 +799,43 @@ workflow_definition:
assert "button:nth-of-type" not in prompt
assert "secret-token" not in prompt
+ def test_runtime_repair_context_prompt_includes_failure_and_page_state(self) -> None:
+ repair_context = CodeAuthoringRepairContext(
+ block_label="search_registry",
+ reason_code="runtime_block_failure",
+ runtime_failure_reason='Timeout waiting for locator("#results")',
+ runtime_failure_class="timeout_waiting_for_selector",
+ failed_block_status="failed",
+ workflow_run_id="wr_failed",
+ current_origin="https://example.test",
+ current_url_present=True,
+ current_title_present=True,
+ page_evidence_source="inspect_page_for_composition",
+ observed_after_workflow_run=True,
+ page_form_summaries=["Search #search"],
+ page_result_summaries=["No results No matching records"],
+ page_action_summaries=["Search button.search disabled"],
+ )
+ ctx = _ctx(
+ block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER,
+ last_code_authoring_repair_context=repair_context,
+ )
+
+ prompt = agent_module._code_authoring_repair_context_prompt(ctx)
+
+ assert 'runtime_failure_reason: Timeout waiting for locator("#results")' in prompt
+ assert "runtime_failure_class: timeout_waiting_for_selector" in prompt
+ assert "failed_block_status: failed" in prompt
+ assert "current_origin: https://example.test" in prompt
+ assert "current_url_present: true" in prompt
+ assert "current_title_present: true" in prompt
+ assert "observed_after_workflow_run: true" in prompt
+ assert "page_forms: Search #search" in prompt
+ assert "page_results: No results No matching records" in prompt
+ assert "page_actions: Search button.search disabled" in prompt
+ assert "adapt the next code block to the observed page state" in prompt
+ assert "do not re-emit the same failing selector or name path" in prompt
+
class TestVerifiedWorkflowOrNone:
"""SKY-9143 strict invariant: a proposal surfaces only after a passing test this turn."""
diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.py
index c9fa8344a..73014b64a 100644
--- a/tests/unit/test_copilot_code_block_persist_seam.py
+++ b/tests/unit/test_copilot_code_block_persist_seam.py
@@ -1990,6 +1990,65 @@ class TestCompiledAuthoringImposition:
assert "Unable to bind synthesized parameter `provider_name`" in result["error"]
assert ctx.workflow_yaml == ""
+ @pytest.mark.asyncio
+ async def test_multi_input_missing_synthesized_parameter_rejects_with_repair_context(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ _enable_imposition(ctx)
+ ctx.scout_trajectory = [
+ {
+ "tool_name": "type_text",
+ "selector": "#confirmation",
+ "source_url": "https://example.com/orders",
+ "typed_length": 10,
+ "role": "textbox",
+ "accessible_name": "Enter Confirmation",
+ "trajectory_index": 0,
+ },
+ {
+ "tool_name": "type_text",
+ "selector": "#postal",
+ "source_url": "https://example.com/orders",
+ "typed_length": 5,
+ "role": "textbox",
+ "accessible_name": "ZIP Code",
+ "trajectory_index": 1,
+ },
+ ]
+ submitted = _yaml(
+ """
+ title: Order lookup
+ workflow_definition:
+ parameters:
+ - {parameter_type: workflow, workflow_parameter_type: string, key: order_lookup}
+ blocks:
+ - block_type: code
+ label: order_status
+ parameter_keys: [order_lookup]
+ code: |
+ await page.locator("#confirmation").fill(str(order_lookup))
+ await page.locator("#postal").fill(str(order_lookup))
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": submitted}, ctx)
+
+ assert result["ok"] is False
+ assert "missing submitted workflow parameter and literal binding is ambiguous" in result["error"]
+ assert ctx.workflow_yaml == ""
+ repair_context = result["data"]["authoring_repair_context"]
+ assert repair_context["reason_code"] == "synthesized_parameter_binding_ambiguous"
+ assert repair_context["block_label"] == "order_status"
+ assert repair_context["unresolved_names"] == ["enter_confirmation"]
+ assert repair_context["available_parameter_keys"] == ["order_lookup"]
+ assert repair_context["binding_candidates"] == ["enter_confirmation", "order_lookup"]
+ assert "update_and_run_blocks" in repair_context["repair_instruction"]
+ assert "secret" not in str(repair_context).lower()
+ assert isinstance(ctx.last_code_authoring_repair_context, CodeAuthoringRepairContext)
+ assert ctx.last_code_authoring_repair_context.model_dump(mode="json") == repair_context
+
@pytest.mark.asyncio
async def test_synthesized_internal_parameter_aliases_to_existing_declared_default(
self, monkeypatch: pytest.MonkeyPatch
diff --git a/tests/unit/test_copilot_diagnosis_repair_contract.py b/tests/unit/test_copilot_diagnosis_repair_contract.py
index 4759b72b2..848fd92c5 100644
--- a/tests/unit/test_copilot_diagnosis_repair_contract.py
+++ b/tests/unit/test_copilot_diagnosis_repair_contract.py
@@ -10,6 +10,7 @@ import pytest
from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
from skyvern.forge.sdk.copilot.code_block_preflight import SANDBOX_UNRESOLVED_NAME_REASON_CODE
from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict
+from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext, CopilotContext
from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
DiagnosisFailureType,
@@ -20,6 +21,10 @@ from skyvern.forge.sdk.copilot.run_outcome import (
TERMINAL_CHALLENGE_BLOCKER_REASON_CODE,
TERMINAL_CHALLENGE_RUN_OUTCOME_REASON_CODE,
)
+from skyvern.forge.sdk.copilot.runtime_authoring_repair import (
+ finalize_runtime_authoring_repair_context_from_page_observation,
+ inject_runtime_authoring_repair_context,
+)
from skyvern.forge.sdk.copilot.tools import run_execution as run_execution_module
from skyvern.forge.sdk.copilot.turn_intent import TurnIntent, TurnIntentAuthority, TurnIntentMode
@@ -129,21 +134,29 @@ def test_contract_shapes_for_failed_suspicious_and_missing_credential_cases() ->
def test_authoring_repair_contexts_have_distinct_structural_root_cause_signatures() -> None:
ambiguous = CodeAuthoringRepairContext(
- block_label="retrieve_resale_demand_document_link",
+ block_label="retrieve_document_link",
reason_code="ambiguous_bare_selector",
selector="button",
refiner_selector="xpath=//button[normalize-space()='View / Download']",
)
sandbox = CodeAuthoringRepairContext(
- block_label="retrieve_resale_demand_document_link",
+ block_label="retrieve_document_link",
reason_code=SANDBOX_UNRESOLVED_NAME_REASON_CODE,
unresolved_names=["row_text", "confirmation_number"],
)
sandbox_reordered = CodeAuthoringRepairContext(
- block_label="retrieve_resale_demand_document_link",
+ block_label="retrieve_document_link",
reason_code=SANDBOX_UNRESOLVED_NAME_REASON_CODE,
unresolved_names=["confirmation_number", "row_text"],
)
+ synthesized_binding = CodeAuthoringRepairContext(
+ block_label="retrieve_document_link",
+ reason_code="synthesized_parameter_binding_ambiguous",
+ unresolved_names=["enter_confirmation"],
+ parameter_keys=["enter_confirmation"],
+ available_parameter_keys=["confirmation_number"],
+ binding_candidates=["enter_confirmation", "confirmation_number"],
+ )
ambiguous_contract = build_diagnosis_repair_contract(
source_tool="update_and_run_blocks",
@@ -160,14 +173,28 @@ def test_authoring_repair_contexts_have_distinct_structural_root_cause_signature
result=_authoring_repair_result(sandbox_reordered),
ctx=_ctx(),
)
+ synthesized_binding_contract = build_diagnosis_repair_contract(
+ source_tool="update_and_run_blocks",
+ result=_authoring_repair_result(synthesized_binding),
+ ctx=_ctx(),
+ )
ambiguous_signature = ambiguous_contract.to_trace_data()["root_cause_signature"]
sandbox_signature = sandbox_contract.to_trace_data()["root_cause_signature"]
+ synthesized_binding_signature = synthesized_binding_contract.to_trace_data()["root_cause_signature"]
assert ambiguous_contract.repair_decision.next_action == RepairNextAction.REPAIR
assert sandbox_contract.repair_decision.next_action == RepairNextAction.REPAIR
+ assert synthesized_binding_contract.repair_decision.next_action == RepairNextAction.REPAIR
+ assert synthesized_binding_contract.repair_decision.target_blocks == ["retrieve_document_link"]
assert ambiguous_signature is not None
assert sandbox_signature is not None
+ assert synthesized_binding_signature is not None
assert ambiguous_signature != sandbox_signature
+ assert synthesized_binding_signature not in {ambiguous_signature, sandbox_signature}
+ assert (
+ synthesized_binding_contract.diagnosis_result.root_cause_identity.error_class
+ == "code_authoring_synthesized_parameter_binding_ambiguous"
+ )
assert sandbox_reordered_contract.to_trace_data()["root_cause_signature"] == sandbox_signature
@@ -197,16 +224,56 @@ def test_missing_required_output_key_repair_identity_uses_structural_context_onl
assert contract.diagnosis_result.root_cause_identity.error_class == "code_authoring_missing_required_output_key"
+def test_runtime_authoring_repair_context_identity_includes_bounded_page_state() -> None:
+ base = CodeAuthoringRepairContext(
+ block_label="search_registry",
+ reason_code="runtime_block_failure",
+ runtime_failure_reason='Timeout waiting for locator("#results")',
+ runtime_failure_class="timeout_waiting_for_selector",
+ failed_block_status="failed",
+ workflow_run_id="wr_failed",
+ current_origin="https://example.test",
+ current_url_present=True,
+ current_title_present=True,
+ page_evidence_source="inspect_page_for_composition",
+ observed_after_workflow_run=True,
+ page_form_summaries=["text input labeled Search"],
+ page_result_summaries=["no results container is visible"],
+ page_action_summaries=["button Search is disabled"],
+ )
+ changed_page = base.model_copy(update={"page_result_summaries": ["results table is visible"]})
+
+ base_contract = build_diagnosis_repair_contract(
+ source_tool="update_and_run_blocks",
+ result=_authoring_repair_result(base),
+ ctx=_ctx(),
+ )
+ changed_page_contract = build_diagnosis_repair_contract(
+ source_tool="update_and_run_blocks",
+ result=_authoring_repair_result(changed_page),
+ ctx=_ctx(),
+ )
+
+ assert base_contract.repair_decision.next_action == RepairNextAction.REPAIR
+ assert (
+ base_contract.to_trace_data()["root_cause_signature"]
+ != changed_page_contract.to_trace_data()["root_cause_signature"]
+ )
+ assert base_contract.diagnosis_result.root_cause_identity.error_class == (
+ "code_authoring_runtime_block_failure_timeout_waiting_for_selector"
+ )
+
+
def test_repair_loop_state_resets_when_authoring_repair_context_identity_changes() -> None:
ctx = _ctx()
ambiguous = CodeAuthoringRepairContext(
- block_label="retrieve_resale_demand_document_link",
+ block_label="retrieve_document_link",
reason_code="ambiguous_bare_selector",
selector="button",
refiner_selector="xpath=//button[normalize-space()='View / Download']",
)
sandbox = CodeAuthoringRepairContext(
- block_label="retrieve_resale_demand_document_link",
+ block_label="retrieve_document_link",
reason_code=SANDBOX_UNRESOLVED_NAME_REASON_CODE,
unresolved_names=["confirmation_number", "row_text"],
)
@@ -234,6 +301,446 @@ def test_repair_loop_state_resets_when_authoring_repair_context_identity_changes
assert getattr(ctx, "blocker_signal", None) is None
+def test_failed_run_finalizes_runtime_authoring_repair_context_after_matching_page_observation() -> None:
+ ctx = _ctx()
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+
+ run_execution_module._record_run_blocks_result(
+ ctx,
+ {
+ "ok": False,
+ "error": "Run failed.",
+ "data": {
+ "workflow_run_id": "wr_failed",
+ "overall_status": "failed",
+ "blocks": [
+ {
+ "label": "search_registry",
+ "status": "failed",
+ "failure_reason": 'Timeout waiting for locator("#results")',
+ }
+ ],
+ },
+ },
+ )
+ pending_context = ctx.pending_code_authoring_runtime_repair_context
+ assert isinstance(pending_context, CodeAuthoringRepairContext)
+ assert pending_context.block_label == "search_registry"
+ assert pending_context.workflow_run_id == "wr_failed"
+ ctx.composition_page_evidence = {
+ "workflow_run_id": "wr_failed",
+ "observed_after_workflow_run": True,
+ "source_tool": "inspect_page_for_composition",
+ "current_url": "https://example.test/search?case=secret",
+ "page_title": "Search results",
+ "forms": [{"label": "Search", "selector": "#search", "input_type": "text"}],
+ "result_containers": [{"label": "No results", "text": "No matching records"}],
+ "navigation_targets": [{"label": "Search", "selector": "button.search", "disabled": True}],
+ }
+ result = {
+ "ok": False,
+ "error": "Run failed.",
+ "data": {"workflow_run_id": "wr_failed", "overall_status": "failed"},
+ }
+
+ inject_runtime_authoring_repair_context(ctx, result)
+
+ repair_context = ctx.last_code_authoring_repair_context
+ assert isinstance(repair_context, CodeAuthoringRepairContext)
+ assert result["data"]["authoring_repair_context"] == repair_context.model_dump(mode="json")
+ assert repair_context.block_label == "search_registry"
+ assert repair_context.runtime_failure_class == "timeout_waiting_for_selector"
+ assert repair_context.current_origin == "https://example.test"
+ assert repair_context.current_url_present is True
+ assert repair_context.current_title_present is True
+ assert repair_context.page_evidence_source == "inspect_page_for_composition"
+ assert repair_context.observed_after_workflow_run is True
+ assert repair_context.page_form_summaries == ["Search #search"]
+ assert repair_context.page_result_summaries == ["No results No matching records"]
+ assert repair_context.page_action_summaries == ["Search button.search disabled"]
+ assert "case=secret" not in repair_context.model_dump_json()
+
+
+def test_failed_run_injects_pending_runtime_authoring_context_before_page_observation() -> None:
+ ctx = _ctx()
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ run_result = {
+ "ok": False,
+ "error": "Run failed.",
+ "data": {
+ "workflow_run_id": "wr_failed",
+ "overall_status": "failed",
+ "blocks": [
+ {
+ "label": "search_registry",
+ "status": "failed",
+ "failure_reason": 'Locator.wait_for: strict mode violation: get_by_text("Order Details")',
+ }
+ ],
+ },
+ }
+
+ run_execution_module._record_run_blocks_result(ctx, run_result)
+ inject_runtime_authoring_repair_context(ctx, run_result)
+
+ raw_context = run_result["data"]["authoring_repair_context"]
+ repair_context = CodeAuthoringRepairContext.model_validate(raw_context)
+ assert repair_context.reason_code == "runtime_block_failure"
+ assert repair_context.block_label == "search_registry"
+ assert repair_context.workflow_run_id == "wr_failed"
+ assert repair_context.runtime_failure_class
+ assert repair_context.observed_after_workflow_run is False
+
+ contract = build_diagnosis_repair_contract(
+ source_tool="update_and_run_blocks",
+ result=run_result,
+ ctx=ctx,
+ workflow_updated=True,
+ )
+
+ assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.REPAIRABLE_BLOCK_FAILURE
+ assert contract.diagnosis_result.root_cause_identity.primary_category == "CODE_AUTHORING_REPAIR"
+ assert contract.diagnosis_result.root_cause_identity.error_class.startswith("code_authoring_runtime_block_failure")
+ assert contract.repair_decision.next_action == RepairNextAction.REPAIR
+ assert contract.repair_decision.target_blocks == ["search_registry"]
+
+
+def test_runtime_authoring_repair_context_suppressed_for_stale_or_successful_runs() -> None:
+ stale_ctx = _ctx()
+ run_execution_module._record_run_blocks_result(
+ stale_ctx,
+ {
+ "ok": False,
+ "data": {
+ "workflow_run_id": "wr_failed",
+ "overall_status": "failed",
+ "blocks": [{"label": "search_registry", "status": "failed", "failure_reason": "Button missing"}],
+ },
+ },
+ )
+ stale_ctx.composition_page_evidence = {
+ "workflow_run_id": "wr_other",
+ "observed_after_workflow_run": True,
+ "source_tool": "inspect_page_for_composition",
+ "current_url": "https://example.test/search",
+ "forms": [{"label": "Search", "selector": "#search"}],
+ }
+ stale_result = {"ok": False, "data": {"workflow_run_id": "wr_failed", "overall_status": "failed"}}
+
+ inject_runtime_authoring_repair_context(stale_ctx, stale_result)
+
+ assert "authoring_repair_context" not in stale_result["data"]
+ assert stale_ctx.last_code_authoring_repair_context is None
+
+ success_ctx = _ctx()
+ success_ctx.last_code_authoring_repair_context = CodeAuthoringRepairContext(
+ block_label="search_registry",
+ reason_code="runtime_block_failure",
+ )
+ run_execution_module._record_run_blocks_result(success_ctx, _clean_completed_result())
+
+ assert success_ctx.last_code_authoring_repair_context is None
+
+
+def test_runtime_authoring_repair_context_does_not_override_terminal_stop() -> None:
+ ctx = _ctx()
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ run_execution_module._record_run_blocks_result(
+ ctx,
+ {
+ "ok": False,
+ "data": {
+ "workflow_run_id": "wr_terminal",
+ "overall_status": "failed",
+ "blocks": [
+ {
+ "label": "search_registry",
+ "status": "failed",
+ "failure_reason": "Browser session not found.",
+ }
+ ],
+ },
+ },
+ )
+ ctx.composition_page_evidence = {
+ "workflow_run_id": "wr_terminal",
+ "observed_after_workflow_run": True,
+ "source_tool": "inspect_page_for_composition",
+ "current_url": "https://example.test/search",
+ "forms": [{"label": "Search", "selector": "#search"}],
+ }
+ result = {
+ "ok": False,
+ "error": "Browser session not found.",
+ "data": {
+ "workflow_run_id": "wr_terminal",
+ "overall_status": "failed",
+ "failure_categories": [{"category": "UNRECOVERABLE_TOOL_ERROR"}],
+ },
+ }
+
+ contract = run_execution_module._record_diagnosis_repair_contract(
+ ctx,
+ source_tool="update_and_run_blocks",
+ result=result,
+ workflow_updated=True,
+ )
+
+ assert "authoring_repair_context" not in result["data"]
+ assert ctx.last_code_authoring_repair_context is None
+ assert contract.repair_decision.next_action == RepairNextAction.STOP
+
+
+def test_runtime_authoring_repair_context_requires_bounded_inspect_evidence() -> None:
+ for evidence_update in (
+ {"source_tool": "evaluate", "forms": [{"label": "Search", "selector": "#search"}]},
+ {"source_tool": "inspect_page_for_composition", "forms": []},
+ ):
+ ctx = _ctx()
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ run_execution_module._record_run_blocks_result(
+ ctx,
+ {
+ "ok": False,
+ "data": {
+ "workflow_run_id": "wr_failed",
+ "overall_status": "failed",
+ "blocks": [{"label": "search_registry", "status": "failed", "failure_reason": "Button missing"}],
+ },
+ },
+ )
+ ctx.composition_page_evidence = {
+ "workflow_run_id": "wr_failed",
+ "observed_after_workflow_run": True,
+ "current_url": "https://example.test/search",
+ **evidence_update,
+ }
+
+ assert finalize_runtime_authoring_repair_context_from_page_observation(ctx) is None
+ assert ctx.last_code_authoring_repair_context is None
+
+
+def test_runtime_authoring_repair_context_suppressed_for_terminal_page_evidence() -> None:
+ ctx = _ctx()
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ run_execution_module._record_run_blocks_result(
+ ctx,
+ {
+ "ok": False,
+ "data": {
+ "workflow_run_id": "wr_failed",
+ "overall_status": "failed",
+ "blocks": [{"label": "search_registry", "status": "failed", "failure_reason": "Search disabled"}],
+ },
+ },
+ )
+ ctx.composition_page_evidence = {
+ "workflow_run_id": "wr_failed",
+ "observed_after_workflow_run": True,
+ "source_tool": "inspect_page_for_composition",
+ "current_url": "https://example.test/search",
+ "challenge_state": {
+ "detected": True,
+ "kind": "human_verification",
+ "requires_human_verification": True,
+ "gates_submit_controls": True,
+ "gated_submit_controls": [{"text": "Search", "disabled": True}],
+ },
+ }
+
+ assert finalize_runtime_authoring_repair_context_from_page_observation(ctx) is None
+ assert ctx.pending_code_authoring_runtime_repair_context is None
+ assert ctx.last_code_authoring_repair_context is None
+
+
+def test_runtime_authoring_repair_context_suppressed_for_authority_ask_and_state_stop() -> None:
+ ask_ctx = _ctx()
+ ask_ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ ask_ctx.turn_intent.authority.may_update_workflow = False
+ run_execution_module._record_run_blocks_result(
+ ask_ctx,
+ {
+ "ok": False,
+ "data": {
+ "workflow_run_id": "wr_ask",
+ "overall_status": "failed",
+ "blocks": [{"label": "search_registry", "status": "failed", "failure_reason": "Button missing"}],
+ },
+ },
+ )
+ ask_ctx.composition_page_evidence = {
+ "workflow_run_id": "wr_ask",
+ "observed_after_workflow_run": True,
+ "source_tool": "inspect_page_for_composition",
+ "current_url": "https://example.test/search",
+ "forms": [{"label": "Search", "selector": "#search"}],
+ }
+ ask_result = {"ok": False, "data": {"workflow_run_id": "wr_ask", "overall_status": "failed"}}
+
+ inject_runtime_authoring_repair_context(ask_ctx, ask_result)
+
+ assert "authoring_repair_context" not in ask_result["data"]
+ assert ask_ctx.last_code_authoring_repair_context is None
+
+ stop_ctx = _ctx()
+ stop_ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ stop_ctx.last_test_non_retriable_nav_error = "net::ERR_NAME_NOT_RESOLVED"
+ run_execution_module._record_run_blocks_result(
+ stop_ctx,
+ {
+ "ok": False,
+ "data": {
+ "workflow_run_id": "wr_stop",
+ "overall_status": "failed",
+ "blocks": [{"label": "open", "status": "failed", "failure_reason": "net::ERR_NAME_NOT_RESOLVED"}],
+ },
+ },
+ )
+ stop_ctx.composition_page_evidence = {
+ "workflow_run_id": "wr_stop",
+ "observed_after_workflow_run": True,
+ "source_tool": "inspect_page_for_composition",
+ "current_url": "https://example.test/search",
+ "forms": [{"label": "Search", "selector": "#search"}],
+ }
+ stop_result = {
+ "ok": False,
+ "error": "Failed to navigate to url https://bad.example.",
+ "data": {"workflow_run_id": "wr_stop", "overall_status": "failed"},
+ }
+
+ inject_runtime_authoring_repair_context(stop_ctx, stop_result)
+
+ assert "authoring_repair_context" not in stop_result["data"]
+ assert stop_ctx.last_code_authoring_repair_context is None
+
+
+def test_direct_runtime_authoring_repair_finalization_suppresses_stop_class_state() -> None:
+ cases = [
+ {
+ "failure_reason": "Failed to navigate to url https://bad.example.",
+ "ctx_attr": ("last_test_non_retriable_nav_error", "net::ERR_NAME_NOT_RESOLVED"),
+ },
+ {
+ "failure_reason": "Browser session not found while taking screenshot.",
+ "ctx_attr": None,
+ },
+ {
+ "failure_reason": "Skipped test run: required credentials are not configured.",
+ "ctx_attr": None,
+ },
+ ]
+ for case in cases:
+ ctx = _ctx()
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ run_execution_module._record_run_blocks_result(
+ ctx,
+ {
+ "ok": False,
+ "data": {
+ "workflow_run_id": "wr_stop",
+ "overall_status": "failed",
+ "blocks": [
+ {
+ "label": "search_registry",
+ "status": "failed",
+ "failure_reason": case["failure_reason"],
+ }
+ ],
+ },
+ },
+ )
+ ctx_attr = case["ctx_attr"]
+ if ctx_attr is not None:
+ setattr(ctx, ctx_attr[0], ctx_attr[1])
+ ctx.composition_page_evidence = {
+ "workflow_run_id": "wr_stop",
+ "observed_after_workflow_run": True,
+ "source_tool": "inspect_page_for_composition",
+ "current_url": "https://example.test/search",
+ "forms": [{"label": "Search", "selector": "#search"}],
+ }
+
+ assert finalize_runtime_authoring_repair_context_from_page_observation(ctx) is None
+ assert ctx.pending_code_authoring_runtime_repair_context is None
+ assert ctx.last_code_authoring_repair_context is None
+
+
+def test_new_pending_runtime_failure_clears_prior_finalized_runtime_context() -> None:
+ ctx = _ctx()
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ ctx.last_code_authoring_repair_context = CodeAuthoringRepairContext(
+ block_label="old_search",
+ reason_code="runtime_block_failure",
+ runtime_failure_reason="Old failure",
+ workflow_run_id="wr_old",
+ current_origin="https://old.example",
+ observed_after_workflow_run=True,
+ page_form_summaries=["Old #search"],
+ )
+
+ run_execution_module._record_run_blocks_result(
+ ctx,
+ {
+ "ok": False,
+ "data": {
+ "workflow_run_id": "wr_new",
+ "overall_status": "failed",
+ "blocks": [{"label": "new_search", "status": "failed", "failure_reason": "New button missing"}],
+ },
+ },
+ )
+
+ pending_context = ctx.pending_code_authoring_runtime_repair_context
+ assert isinstance(pending_context, CodeAuthoringRepairContext)
+ assert pending_context.block_label == "new_search"
+ assert pending_context.workflow_run_id == "wr_new"
+ assert ctx.last_code_authoring_repair_context is None
+
+
+def test_runtime_authoring_repair_context_sanitizes_failure_and_page_summaries() -> None:
+ ctx = _ctx()
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ run_execution_module._record_run_blocks_result(
+ ctx,
+ {
+ "ok": False,
+ "data": {
+ "workflow_run_id": "wr_secret",
+ "overall_status": "failed",
+ "blocks": [
+ {
+ "label": "search_registry",
+ "status": "failed",
+ "failure_reason": "Timeout after entering password=hunter2",
+ }
+ ],
+ },
+ },
+ )
+ ctx.composition_page_evidence = {
+ "workflow_run_id": "wr_secret",
+ "observed_after_workflow_run": True,
+ "source_tool": "inspect_page_for_composition",
+ "current_url": "https://user:[email protected]/search?password=hunter2",
+ "page_title": "Search",
+ "forms": [{"label": "Password password=hunter2", "selector": "#password"}],
+ "result_containers": [{"label": "Result", "text": "token=secret-token"}],
+ }
+ result = {"ok": False, "data": {"workflow_run_id": "wr_secret", "overall_status": "failed"}}
+
+ inject_runtime_authoring_repair_context(ctx, result)
+
+ repair_context = ctx.last_code_authoring_repair_context
+ assert isinstance(repair_context, CodeAuthoringRepairContext)
+ dumped = repair_context.model_dump_json()
+ assert "hunter2" not in dumped
+ assert "secret-token" not in dumped
+ assert "user:secret" not in dumped
+ assert "password=hunter2" not in dumped
+ assert repair_context.current_origin == "https://example.test"
+
+
def test_schema_incompatibility_failure_type_stops_without_repair() -> None:
# SKY-11380: the typed schema-incompatibility reject must route to STOP, never repair,
# so the agent reports the mismatch instead of churning toward repair_ceiling_reached.
diff --git a/tests/unit/test_copilot_evaluate_page_evidence.py b/tests/unit/test_copilot_evaluate_page_evidence.py
index ea1dfde87..d1da0f987 100644
--- a/tests/unit/test_copilot_evaluate_page_evidence.py
+++ b/tests/unit/test_copilot_evaluate_page_evidence.py
@@ -4,7 +4,9 @@ from unittest.mock import MagicMock
import pytest
+from skyvern.forge.sdk.copilot import agent as agent_module
from skyvern.forge.sdk.copilot.build_phase import BuildPhase
+from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
from skyvern.forge.sdk.copilot.result_evidence import LoadedResultCompositionEvidence
@@ -13,6 +15,7 @@ from skyvern.forge.sdk.copilot.tools import (
_inspect_page_for_composition_impl,
_mark_pending_browser_interaction_observation,
)
+from skyvern.forge.sdk.copilot.tools import run_execution as run_execution_module
from skyvern.forge.sdk.copilot.tools.blockers import _tool_loop_error
from skyvern.forge.sdk.copilot.turn_intent import TurnIntent, TurnIntentAuthority, TurnIntentMode
@@ -237,4 +240,79 @@ async def test_target_url_inspection_does_not_navigate_away_from_interaction_evi
"observation_step": 4,
}
assert 'target_url="current_page"' in result["error"]
- assert "observation_step 4" in result["error"]
+
+
+async def test_current_page_inspection_finalizes_runtime_repair_context_for_next_prompt(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ ctx = _ctx()
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ run_execution_module._record_run_blocks_result(
+ ctx,
+ {
+ "ok": False,
+ "data": {
+ "workflow_run_id": "wr_failed",
+ "overall_status": "failed",
+ "blocks": [
+ {
+ "label": "search_registry",
+ "status": "failed",
+ "failure_reason": 'Timeout waiting for locator("#results")',
+ }
+ ],
+ },
+ },
+ )
+
+ async def fallback_page_info(_ctx: CopilotContext) -> tuple[str, str]:
+ return "https://example.test/search?case=secret", "Search"
+
+ async def capture_evidence(
+ _ctx: CopilotContext,
+ *,
+ inspected_url: str,
+ current_url: str,
+ ) -> tuple[dict[str, object], None]:
+ return (
+ {
+ "inspected_url": inspected_url,
+ "current_url": current_url,
+ "page_title": "Search",
+ "source_tool": "inspect_page_for_composition",
+ "forms": [{"label": "Search", "selector": "#search"}],
+ "result_containers": [{"label": "No results", "text": "No matching records"}],
+ "navigation_targets": [],
+ "challenge_controls": [],
+ },
+ None,
+ )
+
+ async def no_completion_verification(*_args: object, **_kwargs: object) -> None:
+ return None
+
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.tools.composition_capture._fallback_page_info",
+ fallback_page_info,
+ )
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.tools.composition_capture._capture_composition_evidence",
+ capture_evidence,
+ )
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.tools.composition_capture._maybe_run_completion_verification_from_page_observation",
+ no_completion_verification,
+ )
+
+ result = await _inspect_page_for_composition_impl(ctx, "current_page")
+ prompt = agent_module._code_authoring_repair_context_prompt(ctx)
+
+ assert result["ok"] is True
+ assert ctx.pending_code_authoring_runtime_repair_context is None
+ assert ctx.last_code_authoring_repair_context is not None
+ assert ctx.last_code_authoring_repair_context.current_origin == "https://example.test"
+ assert ctx.last_code_authoring_repair_context.page_result_summaries == ["No results No matching records"]
+ assert "runtime_failure_class: timeout_waiting_for_selector" in prompt
+ assert "page_results: No results No matching records" in prompt
+ assert "case=secret" not in ctx.last_code_authoring_repair_context.model_dump_json()