Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-11497 Fix Copilot runtime repair context routing (#6858)
Andrew Neilson3de50e3d3e4d
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex 5e691a6f2..9485bff2a 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -623,6 +623,42 @@ def _code_authoring_repair_context_prompt(ctx: CopilotContext | None) -> str:lines.append(f"source_url: {_clean_authoring_repair_prompt_atom(repair_context.source_url)}")if repair_context.refiner_selector:lines.append(f"refiner_selector: {_clean_authoring_repair_prompt_atom(repair_context.refiner_selector)}")+ if repair_context.reason_code == "runtime_block_failure":+ if repair_context.runtime_failure_reason:+ lines.append(+ f"runtime_failure_reason: {_clean_authoring_repair_prompt_atom(repair_context.runtime_failure_reason)}"+ )+ if repair_context.runtime_failure_class:+ lines.append(+ f"runtime_failure_class: {_clean_authoring_repair_prompt_atom(repair_context.runtime_failure_class)}"+ )+ if repair_context.failed_block_status:+ lines.append(+ f"failed_block_status: {_clean_authoring_repair_prompt_atom(repair_context.failed_block_status)}"+ )+ if repair_context.workflow_run_id:+ workflow_run_id = _clean_authoring_repair_prompt_atom(repair_context.workflow_run_id)+ if workflow_run_id:+ lines.append(f"workflow_run_id: {workflow_run_id}")+ if repair_context.current_origin:+ lines.append(f"current_origin: {_clean_authoring_repair_prompt_atom(repair_context.current_origin)}")+ lines.append(f"current_url_present: {str(repair_context.current_url_present).lower()}")+ lines.append(f"current_title_present: {str(repair_context.current_title_present).lower()}")+ if repair_context.page_evidence_source:+ page_evidence_source = _clean_authoring_repair_prompt_atom(repair_context.page_evidence_source)+ if page_evidence_source:+ lines.append(f"page_evidence_source: {page_evidence_source}")+ lines.append(f"observed_after_workflow_run: {str(repair_context.observed_after_workflow_run).lower()}")+ if repair_context.page_form_summaries:+ lines.append(f"page_forms: {_render_authoring_repair_prompt_list(repair_context.page_form_summaries)}")+ if repair_context.page_result_summaries:+ lines.append(f"page_results: {_render_authoring_repair_prompt_list(repair_context.page_result_summaries)}")+ if repair_context.page_action_summaries:+ lines.append(f"page_actions: {_render_authoring_repair_prompt_list(repair_context.page_action_summaries)}")+ if repair_context.page_challenge_summaries:+ lines.append(+ f"page_challenges: {_render_authoring_repair_prompt_list(repair_context.page_challenge_summaries)}"+ )selector_alternative_lines = _render_selector_repair_alternatives(repair_context.selector_alternatives)if selector_alternative_lines:lines.append("same_page_selector_alternatives:")@@ -646,12 +682,29 @@ def _code_authoring_repair_context_prompt(ctx: CopilotContext | None) -> str:"list the exact key in the code block's parameter_keys, reference the exact key as a bare Python ""variable in code, do not hardcode the eval value, and rerun via update_and_run_blocks.")+ if repair_context.reason_code == "synthesized_parameter_binding_ambiguous":+ binding_action_lines = _render_unresolved_name_binding_actions(+ repair_context.unresolved_names, available_parameter_keys+ )+ if binding_action_lines:+ lines.append("binding_actions:")+ lines.extend(binding_action_lines)+ lines.append(+ "For synthesized parameter binding, declare and use the exact workflow input key, include that exact "+ "key in the code block's parameter_keys, reference it as a bare Python variable in code, do not guess "+ "or hardcode the runtime value, and rerun via update_and_run_blocks."+ )if repair_context.reason_code == "ambiguous_bare_selector":lines.append("For ambiguous selectors, do not re-emit the bare selector or a positional nth selector. ""Use the same-page alternatives when they are stable, or re-scout the same page and choose a ""stable role/name/data attribute.")+ if repair_context.reason_code == "runtime_block_failure":+ lines.append(+ "For runtime failures, adapt the next code block to the observed page state and do not re-emit "+ "the same failing selector or name path."+ )lines.append(_clean_authoring_repair_prompt_atom(repair_context.repair_instruction, max_chars=260))return "\n\n" + "\n".join(line for line in lines if line)diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex 9f18f2fa2..d6fefb547 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -147,6 +147,19 @@ class CodeAuthoringRepairContext(BaseModel):selector_alternatives: list[dict[str, str]] = Field(default_factory=list)allowed_global_names: list[str] = Field(default_factory=list)allowed_helper_surface: dict[str, list[str]] = Field(default_factory=dict)+ runtime_failure_reason: str | None = None+ runtime_failure_class: str | None = None+ failed_block_status: str | None = None+ workflow_run_id: str | None = None+ current_origin: str | None = None+ current_url_present: bool = False+ current_title_present: bool = False+ page_evidence_source: str | None = None+ observed_after_workflow_run: bool = False+ page_form_summaries: list[str] = Field(default_factory=list)+ page_result_summaries: list[str] = Field(default_factory=list)+ page_action_summaries: list[str] = Field(default_factory=list)+ page_challenge_summaries: list[str] = Field(default_factory=list)repair_instruction: str = "add workflow-input-like names to parameter_keys, or stop referencing them."@@ -499,6 +512,7 @@ class CopilotContext(AgentContext):# True when the most-recent such rejection deferred to the credential-scout# gate, so the churn backstop yields to that message instead of pre-empting it.last_code_authoring_reject_was_credential_priority: bool = False+ pending_code_authoring_runtime_repair_context: CodeAuthoringRepairContext | None = Nonelast_code_authoring_repair_context: CodeAuthoringRepairContext | None = None# Turn-scoped monotonic marks of verified forward progress: the union of# completion criteria the judge confirmed satisfied so far this turn, and thediff --git a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.pyindex 6bf184051..e6f91b220 100644--- a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py+++ b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py@@ -350,15 +350,31 @@ def _repair_context_root_cause_identity(payload["selector"] = selectorpayload["refiner_selector"] = refiner_selectorselector_kind = "selector" if selector else ""+ elif reason_code == "runtime_block_failure":+ runtime_failure_class = _safe_text(repair_context.runtime_failure_class, 80)+ payload["runtime_failure_class"] = runtime_failure_class+ payload["failed_block_status"] = _safe_text(repair_context.failed_block_status, 80)+ payload["current_origin"] = _safe_text(repair_context.current_origin, 120)+ payload["current_url_present"] = repair_context.current_url_present+ payload["current_title_present"] = repair_context.current_title_present+ payload["page_evidence_source"] = _safe_text(repair_context.page_evidence_source, 80)+ payload["observed_after_workflow_run"] = repair_context.observed_after_workflow_run+ payload["page_form_summaries"] = _safe_identity_list(repair_context.page_form_summaries)+ payload["page_result_summaries"] = _safe_identity_list(repair_context.page_result_summaries)+ payload["page_action_summaries"] = _safe_identity_list(repair_context.page_action_summaries)+ payload["page_challenge_summaries"] = _safe_identity_list(repair_context.page_challenge_summaries)elif repair_context.unresolved_names:payload["unresolved_names"] = _safe_identity_list(repair_context.unresolved_names)+ error_class_suffix = _identity_token(reason_code)+ if reason_code == "runtime_block_failure" and repair_context.runtime_failure_class:+ error_class_suffix = f"{error_class_suffix}_{_identity_token(repair_context.runtime_failure_class)}"serialized = json.dumps(payload, sort_keys=True, separators=(",", ":"))return RepairRootCauseIdentity(root_cause_signature=hashlib.sha256(serialized.encode("utf-8")).hexdigest(),primary_category=_AUTHORING_REPAIR_CATEGORY,failure_categories=(_AUTHORING_REPAIR_CATEGORY,),- error_class=f"code_authoring_{_identity_token(reason_code)}",+ error_class=f"code_authoring_{error_class_suffix}",selector_kind=selector_kind,selector=selector,)diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.pyindex 799450bc5..e4d018d21 100644--- a/skyvern/forge/sdk/copilot/runtime.py+++ b/skyvern/forge/sdk/copilot/runtime.py@@ -204,6 +204,7 @@ class AgentContext:repeated_failure_nudge_emitted_at_streak: int = 0code_authoring_guardrail_reject_count: int = 0last_code_authoring_reject_was_credential_priority: bool = False+ pending_code_authoring_runtime_repair_context: CodeAuthoringRepairContext | None = Nonelast_code_authoring_repair_context: CodeAuthoringRepairContext | None = Nonechallenge_gated_proxy_retry_count: int = 0last_test_non_retriable_nav_error: str | None = Nonediff --git a/skyvern/forge/sdk/copilot/runtime_authoring_repair.py b/skyvern/forge/sdk/copilot/runtime_authoring_repair.pynew file mode 100644index 000000000..d73fea69f--- /dev/null+++ b/skyvern/forge/sdk/copilot/runtime_authoring_repair.py@@ -0,0 +1,318 @@+from __future__ import annotations++import re+from typing import Any, TypeGuard++from skyvern.forge.sdk.copilot.composition_evidence import has_bounded_page_schema+from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, normalize_block_authoring_policy+from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext+from skyvern.forge.sdk.copilot.failure_tracking import ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY+from skyvern.forge.sdk.copilot.request_policy import redact_raw_secrets_for_prompt+from skyvern.forge.sdk.copilot.run_outcome import trusted_terminal_challenge_category_name+from skyvern.forge.sdk.copilot.workflow_credential_utils import url_origin++_RUNTIME_AUTHORING_REASON_CODE = "runtime_block_failure"+_RUNTIME_SUMMARY_MAX_CHARS = 120+_RUNTIME_SUMMARY_MAX_ITEMS = 5+_INSPECT_PAGE_SOURCE_TOOL = "inspect_page_for_composition"+++def is_runtime_authoring_repair_context(repair_context: object) -> TypeGuard[CodeAuthoringRepairContext]:+ return (+ isinstance(repair_context, CodeAuthoringRepairContext)+ and repair_context.reason_code == _RUNTIME_AUTHORING_REASON_CODE+ )+++def clear_runtime_authoring_repair_context(copilot_ctx: Any) -> None:+ copilot_ctx.pending_code_authoring_runtime_repair_context = None+ if is_runtime_authoring_repair_context(getattr(copilot_ctx, "last_code_authoring_repair_context", None)):+ copilot_ctx.last_code_authoring_repair_context = None+++def _bounded_runtime_text(value: Any, max_chars: int = _RUNTIME_SUMMARY_MAX_CHARS) -> str:+ if not isinstance(value, str):+ return ""+ text = redact_raw_secrets_for_prompt(" ".join(value.split()))+ return text[:max_chars]+++def _runtime_failure_class(reason: str) -> str:+ reason_lower = reason.lower()+ if "timeout" in reason_lower and any(token in reason_lower for token in ("locator", "selector", "element")):+ return "timeout_waiting_for_selector"+ if "not found" in reason_lower and any(token in reason_lower for token in ("locator", "selector", "element")):+ return "selector_not_found"+ normalized = re.sub(r"[^a-z0-9]+", "_", reason_lower).strip("_")+ return normalized[:80].strip("_") or "runtime_failure"+++def _origin_from_runtime_url(value: Any) -> str | None:+ if not isinstance(value, str) or not value.strip():+ return None+ return url_origin(value)+++def _runtime_summary_entry(entry: Any, keys: tuple[str, ...]) -> str:+ if not isinstance(entry, dict):+ return _bounded_runtime_text(entry)+ parts = [+ _bounded_runtime_text(entry.get(key), 60)+ if not isinstance(entry.get(key), bool)+ else ("disabled" if entry.get(key) is True else "enabled")+ for key in keys+ ]+ return _bounded_runtime_text(" ".join(part for part in parts if part))+++def _runtime_summary_list(value: Any, keys: tuple[str, ...]) -> list[str]:+ if not isinstance(value, list):+ return []+ summaries: list[str] = []+ for entry in value[:_RUNTIME_SUMMARY_MAX_ITEMS]:+ summary = _runtime_summary_entry(entry, keys)+ if summary:+ summaries.append(summary)+ return summaries+++def _post_run_terminal_page_evidence(evidence: dict[str, Any]) -> bool:+ if evidence.get("observed_after_workflow_run") is not True:+ return False+ challenge_state = evidence.get("challenge_state")+ if isinstance(challenge_state, dict):+ if challenge_state.get("gates_submit_controls") is True:+ return True+ if (+ challenge_state.get("detected") is True+ and challenge_state.get("requires_human_verification") is True+ and _runtime_summary_list(evidence.get("forms"), ("label", "selector"))+ ):+ return True+ indicators = evidence.get("anti_bot_indicators")+ has_indicators = isinstance(indicators, list) and any(isinstance(item, str) and item.strip() for item in indicators)+ controls = evidence.get("challenge_controls")+ has_controls = isinstance(controls, list) and any(isinstance(item, dict) for item in controls)+ return has_indicators and has_controls+++def _first_runtime_failed_block(data: dict[str, Any]) -> dict[str, Any] | None:+ blocks = data.get("blocks")+ if not isinstance(blocks, list):+ return None+ for block in blocks:+ if not isinstance(block, dict):+ continue+ status = str(block.get("status") or "").lower()+ if status in {"failed", "terminated", "canceled", "timed_out"}:+ return block+ return None+++def record_pending_runtime_authoring_repair_context(copilot_ctx: Any, result: dict[str, Any]) -> None:+ if bool(result.get("ok", False)):+ clear_runtime_authoring_repair_context(copilot_ctx)+ return+ data = result.get("data")+ if not isinstance(data, dict):+ clear_runtime_authoring_repair_context(copilot_ctx)+ return+ run_id = data.get("workflow_run_id")+ if not isinstance(run_id, str) or not run_id:+ clear_runtime_authoring_repair_context(copilot_ctx)+ return+ block = _first_runtime_failed_block(data)+ failure_reason = ""+ block_label = _bounded_runtime_text(data.get("frontier_start_label"), 80)+ failed_block_status = _bounded_runtime_text(data.get("overall_status"), 40)+ if block is not None:+ block_label = _bounded_runtime_text(block.get("label"), 80) or block_label+ failed_block_status = _bounded_runtime_text(block.get("status"), 40) or failed_block_status+ failure_reason = _bounded_runtime_text(block.get("failure_reason"), 240)+ failure_reason = failure_reason or _bounded_runtime_text(data.get("failure_reason"), 240)+ failure_reason = failure_reason or _bounded_runtime_text(result.get("error"), 240)+ if not block_label or not failure_reason:+ clear_runtime_authoring_repair_context(copilot_ctx)+ return+ if is_runtime_authoring_repair_context(getattr(copilot_ctx, "last_code_authoring_repair_context", None)):+ copilot_ctx.last_code_authoring_repair_context = None+ copilot_ctx.pending_code_authoring_runtime_repair_context = CodeAuthoringRepairContext(+ block_label=block_label,+ reason_code=_RUNTIME_AUTHORING_REASON_CODE,+ runtime_failure_reason=failure_reason,+ runtime_failure_class=_runtime_failure_class(failure_reason),+ failed_block_status=failed_block_status or None,+ workflow_run_id=run_id,+ repair_instruction=(+ "adapt the next code block to the observed page state and do not re-emit the same failing selector "+ "or name path."+ ),+ )+++def _authority_requires_ask(copilot_ctx: Any) -> bool:+ authority = getattr(getattr(copilot_ctx, "turn_intent", None), "authority", None)+ return getattr(authority, "requires_user_input", False) or getattr(authority, "may_update_workflow", True) is False+++def _policy_allows_runtime_authoring_repair(copilot_ctx: Any) -> bool:+ return normalize_block_authoring_policy(getattr(copilot_ctx, "block_authoring_policy", None)) == (+ BlockAuthoringPolicy.CODE_ONLY_BROWSER+ )+++def _error_text_requires_stop(copilot_ctx: Any, data: dict[str, Any], result: dict[str, Any] | None = None) -> bool:+ if getattr(copilot_ctx, "last_test_non_retriable_nav_error", None):+ return True+ text_values = [data.get("failure_reason"), data.get("skip_reason")]+ if result is not None:+ text_values.append(result.get("error"))+ text = " ".join(str(value).lower() for value in text_values if value)+ return (+ "browser session not found" in text+ or "no browser context" in text+ or ("session not found" in text and "browser" in text)+ or ("404" in text and "browser session" in text)+ )+++def _error_text_requires_ask(data: dict[str, Any], result: dict[str, Any] | None = None) -> bool:+ text_values = [data.get("failure_reason"), data.get("skip_reason"), data.get("failure_type")]+ if result is not None:+ text_values.append(result.get("error"))+ text = " ".join(str(value).lower() for value in text_values if value)+ return (+ "workflow_credential_inputs_unbound" in text+ or "credential inputs unbound" in text+ or "required credentials are not configured" in text+ or "missing_credential_or_init" in text+ )+++def _pending_state_has_stop_or_ask_precedence(copilot_ctx: Any, pending: CodeAuthoringRepairContext) -> bool:+ data = {+ "failure_reason": pending.runtime_failure_reason,+ "skip_reason": pending.runtime_failure_reason,+ "failure_type": pending.runtime_failure_class,+ }+ if _authority_requires_ask(copilot_ctx):+ return True+ return _error_text_requires_stop(copilot_ctx, data) or _error_text_requires_ask(data)+++def _result_has_terminal_or_ask_precedence(copilot_ctx: Any, data: dict[str, Any], result: dict[str, Any]) -> bool:+ if _authority_requires_ask(copilot_ctx):+ return True+ if _error_text_requires_stop(copilot_ctx, data, result):+ return True+ if _error_text_requires_ask(data, result):+ return True+ if data.get("active_run_terminal_evidence_detected") is True:+ return True+ if data.get("skip_reason") == "workflow_credential_inputs_unbound":+ return True+ if data.get("failure_type") in {"schema_incompatibility", "missing_credential_or_init"}:+ return True+ categories = data.get("failure_categories")+ if not isinstance(categories, list):+ return False+ for entry in categories:+ if not isinstance(entry, dict):+ continue+ category = entry.get("category")+ if category in {+ "UNRECOVERABLE_TOOL_ERROR",+ ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,+ "ANTI_BOT_DETECTION",+ }:+ return True+ if trusted_terminal_challenge_category_name(entry):+ return True+ return False+++def _matching_bounded_post_run_inspection(+ copilot_ctx: Any, pending: CodeAuthoringRepairContext+) -> dict[str, Any] | None:+ evidence = getattr(copilot_ctx, "composition_page_evidence", None)+ if not isinstance(evidence, dict):+ return None+ if evidence.get("source_tool") != _INSPECT_PAGE_SOURCE_TOOL:+ clear_runtime_authoring_repair_context(copilot_ctx)+ return None+ if evidence.get("observed_after_workflow_run") is not True:+ clear_runtime_authoring_repair_context(copilot_ctx)+ return None+ run_id = evidence.get("workflow_run_id")+ if not isinstance(run_id, str) or run_id != pending.workflow_run_id:+ clear_runtime_authoring_repair_context(copilot_ctx)+ return None+ if not has_bounded_page_schema(evidence):+ clear_runtime_authoring_repair_context(copilot_ctx)+ return None+ if _post_run_terminal_page_evidence(evidence):+ clear_runtime_authoring_repair_context(copilot_ctx)+ return None+ return evidence+++def finalize_runtime_authoring_repair_context_from_page_observation(+ copilot_ctx: Any,+) -> CodeAuthoringRepairContext | None:+ pending = getattr(copilot_ctx, "pending_code_authoring_runtime_repair_context", None)+ if not is_runtime_authoring_repair_context(pending):+ return None+ if not _policy_allows_runtime_authoring_repair(copilot_ctx) or _pending_state_has_stop_or_ask_precedence(+ copilot_ctx, pending+ ):+ clear_runtime_authoring_repair_context(copilot_ctx)+ return None+ evidence = _matching_bounded_post_run_inspection(copilot_ctx, pending)+ if evidence is None:+ return None+ current_url = evidence.get("current_url") or evidence.get("inspected_url")+ page_title = evidence.get("page_title") or evidence.get("title")+ finalized = pending.model_copy(+ update={+ "current_origin": _origin_from_runtime_url(current_url),+ "current_url_present": isinstance(current_url, str) and bool(current_url.strip()),+ "current_title_present": isinstance(page_title, str) and bool(page_title.strip()),+ "page_evidence_source": _bounded_runtime_text(evidence.get("source_tool"), 80) or None,+ "observed_after_workflow_run": True,+ "page_form_summaries": _runtime_summary_list(evidence.get("forms"), ("label", "selector")),+ "page_result_summaries": _runtime_summary_list(evidence.get("result_containers"), ("label", "text")),+ "page_action_summaries": _runtime_summary_list(+ evidence.get("navigation_targets"), ("label", "selector", "disabled")+ ),+ "page_challenge_summaries": _runtime_summary_list(+ evidence.get("challenge_controls"), ("text", "selector", "disabled")+ ),+ }+ )+ copilot_ctx.last_code_authoring_repair_context = finalized+ copilot_ctx.pending_code_authoring_runtime_repair_context = None+ return finalized+++def inject_runtime_authoring_repair_context(copilot_ctx: Any, result: dict[str, Any]) -> None:+ data = result.get("data")+ if not isinstance(data, dict):+ return+ if _result_has_terminal_or_ask_precedence(copilot_ctx, data, result):+ clear_runtime_authoring_repair_context(copilot_ctx)+ data.pop("authoring_repair_context", None)+ return+ repair_context = finalize_runtime_authoring_repair_context_from_page_observation(copilot_ctx)+ if repair_context is None:+ pending = getattr(copilot_ctx, "pending_code_authoring_runtime_repair_context", None)+ if not is_runtime_authoring_repair_context(pending):+ return+ if not _policy_allows_runtime_authoring_repair(copilot_ctx) or _pending_state_has_stop_or_ask_precedence(+ copilot_ctx, pending+ ):+ clear_runtime_authoring_repair_context(copilot_ctx)+ data.pop("authoring_repair_context", None)+ return+ repair_context = pending+ copilot_ctx.last_code_authoring_repair_context = repair_context+ data["authoring_repair_context"] = repair_context.model_dump(mode="json")diff --git a/skyvern/forge/sdk/copilot/tools/composition_capture.py b/skyvern/forge/sdk/copilot/tools/composition_capture.pyindex ed0cdad30..eb1459aeb 100644--- a/skyvern/forge/sdk/copilot/tools/composition_capture.py+++ b/skyvern/forge/sdk/copilot/tools/composition_capture.py@@ -28,6 +28,9 @@ from skyvern.forge.sdk.copilot.context import CopilotContextfrom skyvern.forge.sdk.copilot.failure_tracking import ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORYfrom skyvern.forge.sdk.copilot.llm_config import resolve_fast_copilot_handlerfrom skyvern.forge.sdk.copilot.loop_detection import record_tool_step_result_for_ctx+from skyvern.forge.sdk.copilot.runtime_authoring_repair import (+ finalize_runtime_authoring_repair_context_from_page_observation,+)from skyvern.forge.sdk.copilot.tracing_setup import copilot_spanfrom ._shared import (@@ -840,6 +843,7 @@ async def _inspect_page_for_composition_impl(if bypass_budget_for_post_run_current_page:copilot_ctx.post_run_current_page_inspection_workflow_run_id = run_idcopilot_ctx.composition_page_evidence = evidence+ finalize_runtime_authoring_repair_context_from_page_observation(copilot_ctx)if (isinstance(run_id, str)and run_iddiff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.pyindex c839d416b..7319d617c 100644--- a/skyvern/forge/sdk/copilot/tools/run_execution.py+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py@@ -83,6 +83,11 @@ from skyvern.forge.sdk.copilot.runtime import (AgentContext,ensure_browser_session,)+from skyvern.forge.sdk.copilot.runtime_authoring_repair import (+ clear_runtime_authoring_repair_context,+ inject_runtime_authoring_repair_context,+ record_pending_runtime_authoring_repair_context,+)from skyvern.forge.sdk.copilot.terminal_predicates import outcome_fully_verifiedfrom skyvern.forge.sdk.copilot.tracing_setup import copilot_spanfrom skyvern.forge.sdk.copilot.turn_halt import (@@ -2003,6 +2008,7 @@ def _record_run_blocks_result(copilot_ctx.post_run_page_observation_workflow_run_id = Nonecopilot_ctx.post_run_page_observation_after_failed_test = Falsecopilot_ctx.post_run_current_page_inspection_workflow_run_id = None+ record_pending_runtime_authoring_repair_context(copilot_ctx, result)structured_blocker = _run_blocks_structured_blocker_message(result, copilot_ctx)anti_bot_match, empty_data_blocks, failure_categories = _analyze_run_blocks(result, copilot_ctx)@@ -2071,6 +2077,7 @@ def _record_run_blocks_result(stash_turn_halt_from_blocker_signal(copilot_ctx, signal, source="run_execution")if terminal_challenge is not None:+ clear_runtime_authoring_repair_context(copilot_ctx)# A structured challenge is the more actionable terminal blocker when# artifact-health evidence and challenge evidence appear in the same run.blocked_verification = _terminal_challenge_completion_verification(@@ -2424,6 +2431,7 @@ def _record_diagnosis_repair_contract(result: dict[str, Any],workflow_updated: bool = False,) -> DiagnosisRepairContract:+ inject_runtime_authoring_repair_context(copilot_ctx, result)contract = build_diagnosis_repair_contract(source_tool=source_tool,result=result,diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.pyindex 2ef497508..9c57479c2 100644--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py@@ -379,12 +379,14 @@ def _normalize_code_artifact_metadata(*,impose_defaults: bool = False,scout_trajectory: list[ScoutedInteraction] | None = None,+ verified_runtime_output_paths_by_label: Mapping[str, set[str]] | None = None,) -> tuple[dict[str, dict[str, Any]], str | None]:result = _normalize_code_artifact_metadata_detailed(raw_metadata,workflow_yaml,impose_defaults=impose_defaults,scout_trajectory=scout_trajectory,+ verified_runtime_output_paths_by_label=verified_runtime_output_paths_by_label,)return result.normalized, result.error@@ -395,6 +397,7 @@ def _normalize_code_artifact_metadata_detailed(*,impose_defaults: bool = False,scout_trajectory: list[ScoutedInteraction] | None = None,+ verified_runtime_output_paths_by_label: Mapping[str, set[str]] | None = None,) -> CodeArtifactNormalization:"""Normalize submitted artifact metadata at the persist seam.@@ -506,7 +509,14 @@ def _normalize_code_artifact_metadata_detailed(schema_conformance_error = _extraction_schema_conformance_error(label, dumped, block_code)if schema_conformance_error is not None:item_violations.append(schema_conformance_error)- schema_incompatibility = _extraction_schema_incompatibility(label, dumped, block_code)+ schema_incompatibility = _extraction_schema_incompatibility(+ label,+ dumped,+ block_code,+ verified_runtime_output_paths=(+ verified_runtime_output_paths_by_label.get(label) if verified_runtime_output_paths_by_label else None+ ),+ )if schema_incompatibility is not None:schema_incompatibilities.append(schema_incompatibility)item_violations.append(render_schema_incompatibility_agent_steer(schema_incompatibility))@@ -2405,6 +2415,33 @@ class _SynthesizedParameterReconciliation(NamedTuple):violations: list[str]aliases: dict[str, str]expressions: dict[str, str]+ repair_context: CodeAuthoringRepairContext | None = None+++_SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_REASON_CODE = "synthesized_parameter_binding_ambiguous"+++def _synthesized_parameter_binding_repair_context(+ *,+ parsed: Mapping[str, Any],+ code_block: Mapping[str, Any],+ synthesized_key: str,+ parameter_keys: list[str],+) -> CodeAuthoringRepairContext:+ available_parameter_keys = sorted(_declared_string_workflow_parameter_keys(parsed))+ binding_candidates = [synthesized_key] + [key for key in available_parameter_keys if key != synthesized_key]+ return CodeAuthoringRepairContext(+ block_label=str(code_block.get("label") or ""),+ reason_code=_SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_REASON_CODE,+ unresolved_names=[synthesized_key],+ parameter_keys=list(parameter_keys),+ available_parameter_keys=available_parameter_keys,+ binding_candidates=binding_candidates,+ repair_instruction=(+ f"Declare and use workflow string parameter `{synthesized_key}` exactly, include it in parameter_keys, "+ "reference it as a bare Python variable in code, and rerun via update_and_run_blocks."+ ),+ )def _reconcile_synthesized_parameters(@@ -2437,6 +2474,7 @@ def _reconcile_synthesized_parameters(violations: list[str] = []aliases: dict[str, str] = {}expressions: dict[str, str] = {}+ repair_context: CodeAuthoringRepairContext | None = Nonenon_credential_synthesized = [param for param in synthesized_parameters if not param.get("credential_id")]typed_lengths = [int(interaction.get("typed_length") or 0)@@ -2573,6 +2611,13 @@ def _reconcile_synthesized_parameters(violations.append(f"Unable to bind synthesized parameter `{key}`: missing submitted workflow parameter and literal binding is ambiguous.")+ if repair_context is None:+ repair_context = _synthesized_parameter_binding_repair_context(+ parsed=parsed,+ code_block=code_block,+ synthesized_key=key,+ parameter_keys=parameter_keys,+ )continuetyped_length = typed_lengths[0] if len(typed_lengths) == 1 else None@@ -2618,7 +2663,7 @@ def _reconcile_synthesized_parameters(code_block["parameter_keys"] = parameter_keys# Submitted synthesized parameter rows are re-derived from scout evidence before persistence.code_block.pop("parameters", None)- return _SynthesizedParameterReconciliation(parameter_keys, violations, aliases, expressions)+ return _SynthesizedParameterReconciliation(parameter_keys, violations, aliases, expressions, repair_context)def _maybe_impose_synthesized_code_block(workflow_yaml: str, ctx: AgentContext) -> _SynthesizedCodeImpositionResult:@@ -2702,6 +2747,8 @@ def _maybe_impose_synthesized_code_block(workflow_yaml: str, ctx: AgentContext)scout_trajectory=scout_trajectory,)violations.extend(parameter_reconciliation.violations)+ if repair_context is None:+ repair_context = parameter_reconciliation.repair_contextif violations:return _SynthesizedCodeImpositionResult(workflow_yaml=workflow_yaml,@@ -3136,6 +3183,51 @@ def _top_level_path_segment(path: str) -> str:return head.strip()+_STRUCTURAL_RUNTIME_OUTPUT_KEY_RE = re.compile(r"^[a-z]+(?:_[a-z]+)*(?:_[0-9])?$")+_SENSITIVE_RUNTIME_OUTPUT_KEY_TERMS = frozenset(+ {"api_key", "access_key", "password", "secret", "token", "credential", "email"}+)+++def _is_structural_runtime_output_key(key: str) -> bool:+ return (+ _STRUCTURAL_RUNTIME_OUTPUT_KEY_RE.fullmatch(key) is not None+ and not keyword.iskeyword(key)+ and key not in _SENSITIVE_RUNTIME_OUTPUT_KEY_TERMS+ and not any(part in _SENSITIVE_RUNTIME_OUTPUT_KEY_TERMS for part in key.split("_"))+ )+++def _verified_runtime_output_contract_paths(value: object, *, prefix: str = "") -> set[str]:+ paths: set[str] = set()+ if isinstance(value, Mapping):+ for raw_key, child in value.items():+ if not isinstance(raw_key, str):+ continue+ key = raw_key.strip()+ if not _is_structural_runtime_output_key(key):+ continue+ path = f"{prefix}.{key}" if prefix else key+ paths.add(path)+ paths |= _verified_runtime_output_contract_paths(child, prefix=path)+ return paths+ if isinstance(value, list):+ for item in value:+ paths |= _verified_runtime_output_contract_paths(item, prefix=prefix)+ return paths+++def _verified_runtime_output_contract_paths_by_label(ctx: AgentContext, workflow_yaml: str) -> dict[str, set[str]]:+ if _copilot_block_authoring_policy(ctx) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:+ return {}+ code_block_labels = set(_workflow_yaml_code_blocks_by_label(workflow_yaml))+ return {+ label: paths+ for label, output in ctx.verified_block_outputs.items()+ if label in code_block_labels and (paths := _verified_runtime_output_contract_paths(output))+ }++def _known_output_contract_paths(artifact: Mapping[str, Any], code: str) -> set[str]:"""Top-level field names the block is known to produce: the snippet's return-dictkeys plus the confirmed `goal_value_paths`' top-level segments. Empty when neither@@ -3162,7 +3254,11 @@ def _schema_property_summary(schema: Mapping[str, Any]) -> str:def _extraction_schema_incompatibility(- label: str, artifact: Mapping[str, Any], code: str+ label: str,+ artifact: Mapping[str, Any],+ code: str,+ *,+ verified_runtime_output_paths: set[str] | None = None,) -> SchemaIncompatibility | None:"""Detect an edited `extraction_schema` whose object property names overlap NONE ofthe block's known output contract. Unlike `_extraction_schema_conformance_error`,@@ -3172,6 +3268,8 @@ def _extraction_schema_incompatibility(if _is_download_intent(artifact, code) or not code.strip():return Noneknown_paths = _known_output_contract_paths(artifact, code)+ if verified_runtime_output_paths:+ known_paths |= {_top_level_path_segment(path) for path in verified_runtime_output_paths}if not known_paths:return Noneincompatible: set[str] = set()@@ -3713,6 +3811,7 @@ async def _update_workflow(workflow_yaml,impose_defaults=_copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER,scout_trajectory=scout_trajectory if isinstance(scout_trajectory, list) else None,+ verified_runtime_output_paths_by_label=_verified_runtime_output_contract_paths_by_label(ctx, workflow_yaml),)code_artifact_metadata = normalization.normalizedcode_artifact_metadata_error = normalization.errordiff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.pyindex e9e0728f1..852c6fa43 100644--- a/tests/unit/test_copilot_agent_helpers.py+++ b/tests/unit/test_copilot_agent_helpers.py@@ -724,6 +724,30 @@ workflow_definition:assert "do not hardcode the eval value" in promptassert "rerun via update_and_run_blocks" in prompt+ def test_synthesized_parameter_binding_prompt_uses_exact_key(self) -> None:+ repair_context = CodeAuthoringRepairContext(+ block_label="order_status",+ reason_code="synthesized_parameter_binding_ambiguous",+ unresolved_names=["enter_confirmation"],+ parameter_keys=["enter_confirmation"],+ available_parameter_keys=["confirmation_number"],+ binding_candidates=["enter_confirmation", "confirmation_number"],+ )+ ctx = _ctx(+ block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER,+ last_code_authoring_repair_context=repair_context,+ )++ prompt = agent_module._code_authoring_repair_context_prompt(ctx)++ assert "reason_code: synthesized_parameter_binding_ambiguous" in prompt+ assert "binding_candidates: enter_confirmation, confirmation_number" in prompt+ assert "declare and use the exact workflow input key" in prompt+ assert "include that exact key in the code block's parameter_keys" in prompt+ assert "reference it as a bare Python variable in code" in prompt+ assert "do not guess or hardcode the runtime value" in prompt+ assert "rerun via update_and_run_blocks" in prompt+def test_ambiguous_selector_repair_context_prompt_includes_selector_details(self) -> None:repair_context = CodeAuthoringRepairContext(block_label="order_status",@@ -775,6 +799,43 @@ workflow_definition:assert "button:nth-of-type" not in promptassert "secret-token" not in prompt+ def test_runtime_repair_context_prompt_includes_failure_and_page_state(self) -> None:+ repair_context = CodeAuthoringRepairContext(+ block_label="search_registry",+ reason_code="runtime_block_failure",+ runtime_failure_reason='Timeout waiting for locator("#results")',+ runtime_failure_class="timeout_waiting_for_selector",+ failed_block_status="failed",+ workflow_run_id="wr_failed",+ current_origin="https://example.test",+ current_url_present=True,+ current_title_present=True,+ page_evidence_source="inspect_page_for_composition",+ observed_after_workflow_run=True,+ page_form_summaries=["Search #search"],+ page_result_summaries=["No results No matching records"],+ page_action_summaries=["Search button.search disabled"],+ )+ ctx = _ctx(+ block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER,+ last_code_authoring_repair_context=repair_context,+ )++ prompt = agent_module._code_authoring_repair_context_prompt(ctx)++ assert 'runtime_failure_reason: Timeout waiting for locator("#results")' in prompt+ assert "runtime_failure_class: timeout_waiting_for_selector" in prompt+ assert "failed_block_status: failed" in prompt+ assert "current_origin: https://example.test" in prompt+ assert "current_url_present: true" in prompt+ assert "current_title_present: true" in prompt+ assert "observed_after_workflow_run: true" in prompt+ assert "page_forms: Search #search" in prompt+ assert "page_results: No results No matching records" in prompt+ assert "page_actions: Search button.search disabled" in prompt+ assert "adapt the next code block to the observed page state" in prompt+ assert "do not re-emit the same failing selector or name path" in prompt+class TestVerifiedWorkflowOrNone:"""SKY-9143 strict invariant: a proposal surfaces only after a passing test this turn."""diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.pyindex c9fa8344a..73014b64a 100644--- a/tests/unit/test_copilot_code_block_persist_seam.py+++ b/tests/unit/test_copilot_code_block_persist_seam.py@@ -1990,6 +1990,65 @@ class TestCompiledAuthoringImposition:assert "Unable to bind synthesized parameter `provider_name`" in result["error"]assert ctx.workflow_yaml == ""+ @pytest.mark.asyncio+ async def test_multi_input_missing_synthesized_parameter_rejects_with_repair_context(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ _stub_successful_update(monkeypatch)+ ctx = _code_only_ctx()+ _enable_imposition(ctx)+ ctx.scout_trajectory = [+ {+ "tool_name": "type_text",+ "selector": "#confirmation",+ "source_url": "https://example.com/orders",+ "typed_length": 10,+ "role": "textbox",+ "accessible_name": "Enter Confirmation",+ "trajectory_index": 0,+ },+ {+ "tool_name": "type_text",+ "selector": "#postal",+ "source_url": "https://example.com/orders",+ "typed_length": 5,+ "role": "textbox",+ "accessible_name": "ZIP Code",+ "trajectory_index": 1,+ },+ ]+ submitted = _yaml(+ """+ title: Order lookup+ workflow_definition:+ parameters:+ - {parameter_type: workflow, workflow_parameter_type: string, key: order_lookup}+ blocks:+ - block_type: code+ label: order_status+ parameter_keys: [order_lookup]+ code: |+ await page.locator("#confirmation").fill(str(order_lookup))+ await page.locator("#postal").fill(str(order_lookup))+ """+ )++ result = await _update_workflow({"workflow_yaml": submitted}, ctx)++ assert result["ok"] is False+ assert "missing submitted workflow parameter and literal binding is ambiguous" in result["error"]+ assert ctx.workflow_yaml == ""+ repair_context = result["data"]["authoring_repair_context"]+ assert repair_context["reason_code"] == "synthesized_parameter_binding_ambiguous"+ assert repair_context["block_label"] == "order_status"+ assert repair_context["unresolved_names"] == ["enter_confirmation"]+ assert repair_context["available_parameter_keys"] == ["order_lookup"]+ assert repair_context["binding_candidates"] == ["enter_confirmation", "order_lookup"]+ assert "update_and_run_blocks" in repair_context["repair_instruction"]+ assert "secret" not in str(repair_context).lower()+ assert isinstance(ctx.last_code_authoring_repair_context, CodeAuthoringRepairContext)+ assert ctx.last_code_authoring_repair_context.model_dump(mode="json") == repair_context+@pytest.mark.asyncioasync def test_synthesized_internal_parameter_aliases_to_existing_declared_default(self, monkeypatch: pytest.MonkeyPatchdiff --git a/tests/unit/test_copilot_diagnosis_repair_contract.py b/tests/unit/test_copilot_diagnosis_repair_contract.pyindex 4759b72b2..848fd92c5 100644--- a/tests/unit/test_copilot_diagnosis_repair_contract.py+++ b/tests/unit/test_copilot_diagnosis_repair_contract.py@@ -10,6 +10,7 @@ import pytestfrom skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignalfrom skyvern.forge.sdk.copilot.code_block_preflight import SANDBOX_UNRESOLVED_NAME_REASON_CODEfrom skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict+from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicyfrom skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext, CopilotContextfrom skyvern.forge.sdk.copilot.diagnosis_repair_contract import (DiagnosisFailureType,@@ -20,6 +21,10 @@ from skyvern.forge.sdk.copilot.run_outcome import (TERMINAL_CHALLENGE_BLOCKER_REASON_CODE,TERMINAL_CHALLENGE_RUN_OUTCOME_REASON_CODE,)+from skyvern.forge.sdk.copilot.runtime_authoring_repair import (+ finalize_runtime_authoring_repair_context_from_page_observation,+ inject_runtime_authoring_repair_context,+)from skyvern.forge.sdk.copilot.tools import run_execution as run_execution_modulefrom skyvern.forge.sdk.copilot.turn_intent import TurnIntent, TurnIntentAuthority, TurnIntentMode@@ -129,21 +134,29 @@ def test_contract_shapes_for_failed_suspicious_and_missing_credential_cases() ->def test_authoring_repair_contexts_have_distinct_structural_root_cause_signatures() -> None:ambiguous = CodeAuthoringRepairContext(- block_label="retrieve_resale_demand_document_link",+ block_label="retrieve_document_link",reason_code="ambiguous_bare_selector",selector="button",refiner_selector="xpath=//button[normalize-space()='View / Download']",)sandbox = CodeAuthoringRepairContext(- block_label="retrieve_resale_demand_document_link",+ block_label="retrieve_document_link",reason_code=SANDBOX_UNRESOLVED_NAME_REASON_CODE,unresolved_names=["row_text", "confirmation_number"],)sandbox_reordered = CodeAuthoringRepairContext(- block_label="retrieve_resale_demand_document_link",+ block_label="retrieve_document_link",reason_code=SANDBOX_UNRESOLVED_NAME_REASON_CODE,unresolved_names=["confirmation_number", "row_text"],)+ synthesized_binding = CodeAuthoringRepairContext(+ block_label="retrieve_document_link",+ reason_code="synthesized_parameter_binding_ambiguous",+ unresolved_names=["enter_confirmation"],+ parameter_keys=["enter_confirmation"],+ available_parameter_keys=["confirmation_number"],+ binding_candidates=["enter_confirmation", "confirmation_number"],+ )ambiguous_contract = build_diagnosis_repair_contract(source_tool="update_and_run_blocks",@@ -160,14 +173,28 @@ def test_authoring_repair_contexts_have_distinct_structural_root_cause_signatureresult=_authoring_repair_result(sandbox_reordered),ctx=_ctx(),)+ synthesized_binding_contract = build_diagnosis_repair_contract(+ source_tool="update_and_run_blocks",+ result=_authoring_repair_result(synthesized_binding),+ ctx=_ctx(),+ )ambiguous_signature = ambiguous_contract.to_trace_data()["root_cause_signature"]sandbox_signature = sandbox_contract.to_trace_data()["root_cause_signature"]+ synthesized_binding_signature = synthesized_binding_contract.to_trace_data()["root_cause_signature"]assert ambiguous_contract.repair_decision.next_action == RepairNextAction.REPAIRassert sandbox_contract.repair_decision.next_action == RepairNextAction.REPAIR+ assert synthesized_binding_contract.repair_decision.next_action == RepairNextAction.REPAIR+ assert synthesized_binding_contract.repair_decision.target_blocks == ["retrieve_document_link"]assert ambiguous_signature is not Noneassert sandbox_signature is not None+ assert synthesized_binding_signature is not Noneassert ambiguous_signature != sandbox_signature+ assert synthesized_binding_signature not in {ambiguous_signature, sandbox_signature}+ assert (+ synthesized_binding_contract.diagnosis_result.root_cause_identity.error_class+ == "code_authoring_synthesized_parameter_binding_ambiguous"+ )assert sandbox_reordered_contract.to_trace_data()["root_cause_signature"] == sandbox_signature@@ -197,16 +224,56 @@ def test_missing_required_output_key_repair_identity_uses_structural_context_onlassert contract.diagnosis_result.root_cause_identity.error_class == "code_authoring_missing_required_output_key"+def test_runtime_authoring_repair_context_identity_includes_bounded_page_state() -> None:+ base = CodeAuthoringRepairContext(+ block_label="search_registry",+ reason_code="runtime_block_failure",+ runtime_failure_reason='Timeout waiting for locator("#results")',+ runtime_failure_class="timeout_waiting_for_selector",+ failed_block_status="failed",+ workflow_run_id="wr_failed",+ current_origin="https://example.test",+ current_url_present=True,+ current_title_present=True,+ page_evidence_source="inspect_page_for_composition",+ observed_after_workflow_run=True,+ page_form_summaries=["text input labeled Search"],+ page_result_summaries=["no results container is visible"],+ page_action_summaries=["button Search is disabled"],+ )+ changed_page = base.model_copy(update={"page_result_summaries": ["results table is visible"]})++ base_contract = build_diagnosis_repair_contract(+ source_tool="update_and_run_blocks",+ result=_authoring_repair_result(base),+ ctx=_ctx(),+ )+ changed_page_contract = build_diagnosis_repair_contract(+ source_tool="update_and_run_blocks",+ result=_authoring_repair_result(changed_page),+ ctx=_ctx(),+ )++ assert base_contract.repair_decision.next_action == RepairNextAction.REPAIR+ assert (+ base_contract.to_trace_data()["root_cause_signature"]+ != changed_page_contract.to_trace_data()["root_cause_signature"]+ )+ assert base_contract.diagnosis_result.root_cause_identity.error_class == (+ "code_authoring_runtime_block_failure_timeout_waiting_for_selector"+ )++def test_repair_loop_state_resets_when_authoring_repair_context_identity_changes() -> None:ctx = _ctx()ambiguous = CodeAuthoringRepairContext(- block_label="retrieve_resale_demand_document_link",+ block_label="retrieve_document_link",reason_code="ambiguous_bare_selector",selector="button",refiner_selector="xpath=//button[normalize-space()='View / Download']",)sandbox = CodeAuthoringRepairContext(- block_label="retrieve_resale_demand_document_link",+ block_label="retrieve_document_link",reason_code=SANDBOX_UNRESOLVED_NAME_REASON_CODE,unresolved_names=["confirmation_number", "row_text"],)@@ -234,6 +301,446 @@ def test_repair_loop_state_resets_when_authoring_repair_context_identity_changesassert getattr(ctx, "blocker_signal", None) is None+def test_failed_run_finalizes_runtime_authoring_repair_context_after_matching_page_observation() -> None:+ ctx = _ctx()+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER++ run_execution_module._record_run_blocks_result(+ ctx,+ {+ "ok": False,+ "error": "Run failed.",+ "data": {+ "workflow_run_id": "wr_failed",+ "overall_status": "failed",+ "blocks": [+ {+ "label": "search_registry",+ "status": "failed",+ "failure_reason": 'Timeout waiting for locator("#results")',+ }+ ],+ },+ },+ )+ pending_context = ctx.pending_code_authoring_runtime_repair_context+ assert isinstance(pending_context, CodeAuthoringRepairContext)+ assert pending_context.block_label == "search_registry"+ assert pending_context.workflow_run_id == "wr_failed"+ ctx.composition_page_evidence = {+ "workflow_run_id": "wr_failed",+ "observed_after_workflow_run": True,+ "source_tool": "inspect_page_for_composition",+ "current_url": "https://example.test/search?case=secret",+ "page_title": "Search results",+ "forms": [{"label": "Search", "selector": "#search", "input_type": "text"}],+ "result_containers": [{"label": "No results", "text": "No matching records"}],+ "navigation_targets": [{"label": "Search", "selector": "button.search", "disabled": True}],+ }+ result = {+ "ok": False,+ "error": "Run failed.",+ "data": {"workflow_run_id": "wr_failed", "overall_status": "failed"},+ }++ inject_runtime_authoring_repair_context(ctx, result)++ repair_context = ctx.last_code_authoring_repair_context+ assert isinstance(repair_context, CodeAuthoringRepairContext)+ assert result["data"]["authoring_repair_context"] == repair_context.model_dump(mode="json")+ assert repair_context.block_label == "search_registry"+ assert repair_context.runtime_failure_class == "timeout_waiting_for_selector"+ assert repair_context.current_origin == "https://example.test"+ assert repair_context.current_url_present is True+ assert repair_context.current_title_present is True+ assert repair_context.page_evidence_source == "inspect_page_for_composition"+ assert repair_context.observed_after_workflow_run is True+ assert repair_context.page_form_summaries == ["Search #search"]+ assert repair_context.page_result_summaries == ["No results No matching records"]+ assert repair_context.page_action_summaries == ["Search button.search disabled"]+ assert "case=secret" not in repair_context.model_dump_json()+++def test_failed_run_injects_pending_runtime_authoring_context_before_page_observation() -> None:+ ctx = _ctx()+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER+ run_result = {+ "ok": False,+ "error": "Run failed.",+ "data": {+ "workflow_run_id": "wr_failed",+ "overall_status": "failed",+ "blocks": [+ {+ "label": "search_registry",+ "status": "failed",+ "failure_reason": 'Locator.wait_for: strict mode violation: get_by_text("Order Details")',+ }+ ],+ },+ }++ run_execution_module._record_run_blocks_result(ctx, run_result)+ inject_runtime_authoring_repair_context(ctx, run_result)++ raw_context = run_result["data"]["authoring_repair_context"]+ repair_context = CodeAuthoringRepairContext.model_validate(raw_context)+ assert repair_context.reason_code == "runtime_block_failure"+ assert repair_context.block_label == "search_registry"+ assert repair_context.workflow_run_id == "wr_failed"+ assert repair_context.runtime_failure_class+ assert repair_context.observed_after_workflow_run is False++ contract = build_diagnosis_repair_contract(+ source_tool="update_and_run_blocks",+ result=run_result,+ ctx=ctx,+ workflow_updated=True,+ )++ assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.REPAIRABLE_BLOCK_FAILURE+ assert contract.diagnosis_result.root_cause_identity.primary_category == "CODE_AUTHORING_REPAIR"+ assert contract.diagnosis_result.root_cause_identity.error_class.startswith("code_authoring_runtime_block_failure")+ assert contract.repair_decision.next_action == RepairNextAction.REPAIR+ assert contract.repair_decision.target_blocks == ["search_registry"]+++def test_runtime_authoring_repair_context_suppressed_for_stale_or_successful_runs() -> None:+ stale_ctx = _ctx()+ run_execution_module._record_run_blocks_result(+ stale_ctx,+ {+ "ok": False,+ "data": {+ "workflow_run_id": "wr_failed",+ "overall_status": "failed",+ "blocks": [{"label": "search_registry", "status": "failed", "failure_reason": "Button missing"}],+ },+ },+ )+ stale_ctx.composition_page_evidence = {+ "workflow_run_id": "wr_other",+ "observed_after_workflow_run": True,+ "source_tool": "inspect_page_for_composition",+ "current_url": "https://example.test/search",+ "forms": [{"label": "Search", "selector": "#search"}],+ }+ stale_result = {"ok": False, "data": {"workflow_run_id": "wr_failed", "overall_status": "failed"}}++ inject_runtime_authoring_repair_context(stale_ctx, stale_result)++ assert "authoring_repair_context" not in stale_result["data"]+ assert stale_ctx.last_code_authoring_repair_context is None++ success_ctx = _ctx()+ success_ctx.last_code_authoring_repair_context = CodeAuthoringRepairContext(+ block_label="search_registry",+ reason_code="runtime_block_failure",+ )+ run_execution_module._record_run_blocks_result(success_ctx, _clean_completed_result())++ assert success_ctx.last_code_authoring_repair_context is None+++def test_runtime_authoring_repair_context_does_not_override_terminal_stop() -> None:+ ctx = _ctx()+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER+ run_execution_module._record_run_blocks_result(+ ctx,+ {+ "ok": False,+ "data": {+ "workflow_run_id": "wr_terminal",+ "overall_status": "failed",+ "blocks": [+ {+ "label": "search_registry",+ "status": "failed",+ "failure_reason": "Browser session not found.",+ }+ ],+ },+ },+ )+ ctx.composition_page_evidence = {+ "workflow_run_id": "wr_terminal",+ "observed_after_workflow_run": True,+ "source_tool": "inspect_page_for_composition",+ "current_url": "https://example.test/search",+ "forms": [{"label": "Search", "selector": "#search"}],+ }+ result = {+ "ok": False,+ "error": "Browser session not found.",+ "data": {+ "workflow_run_id": "wr_terminal",+ "overall_status": "failed",+ "failure_categories": [{"category": "UNRECOVERABLE_TOOL_ERROR"}],+ },+ }++ contract = run_execution_module._record_diagnosis_repair_contract(+ ctx,+ source_tool="update_and_run_blocks",+ result=result,+ workflow_updated=True,+ )++ assert "authoring_repair_context" not in result["data"]+ assert ctx.last_code_authoring_repair_context is None+ assert contract.repair_decision.next_action == RepairNextAction.STOP+++def test_runtime_authoring_repair_context_requires_bounded_inspect_evidence() -> None:+ for evidence_update in (+ {"source_tool": "evaluate", "forms": [{"label": "Search", "selector": "#search"}]},+ {"source_tool": "inspect_page_for_composition", "forms": []},+ ):+ ctx = _ctx()+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER+ run_execution_module._record_run_blocks_result(+ ctx,+ {+ "ok": False,+ "data": {+ "workflow_run_id": "wr_failed",+ "overall_status": "failed",+ "blocks": [{"label": "search_registry", "status": "failed", "failure_reason": "Button missing"}],+ },+ },+ )+ ctx.composition_page_evidence = {+ "workflow_run_id": "wr_failed",+ "observed_after_workflow_run": True,+ "current_url": "https://example.test/search",+ **evidence_update,+ }++ assert finalize_runtime_authoring_repair_context_from_page_observation(ctx) is None+ assert ctx.last_code_authoring_repair_context is None+++def test_runtime_authoring_repair_context_suppressed_for_terminal_page_evidence() -> None:+ ctx = _ctx()+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER+ run_execution_module._record_run_blocks_result(+ ctx,+ {+ "ok": False,+ "data": {+ "workflow_run_id": "wr_failed",+ "overall_status": "failed",+ "blocks": [{"label": "search_registry", "status": "failed", "failure_reason": "Search disabled"}],+ },+ },+ )+ ctx.composition_page_evidence = {+ "workflow_run_id": "wr_failed",+ "observed_after_workflow_run": True,+ "source_tool": "inspect_page_for_composition",+ "current_url": "https://example.test/search",+ "challenge_state": {+ "detected": True,+ "kind": "human_verification",+ "requires_human_verification": True,+ "gates_submit_controls": True,+ "gated_submit_controls": [{"text": "Search", "disabled": True}],+ },+ }++ assert finalize_runtime_authoring_repair_context_from_page_observation(ctx) is None+ assert ctx.pending_code_authoring_runtime_repair_context is None+ assert ctx.last_code_authoring_repair_context is None+++def test_runtime_authoring_repair_context_suppressed_for_authority_ask_and_state_stop() -> None:+ ask_ctx = _ctx()+ ask_ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER+ ask_ctx.turn_intent.authority.may_update_workflow = False+ run_execution_module._record_run_blocks_result(+ ask_ctx,+ {+ "ok": False,+ "data": {+ "workflow_run_id": "wr_ask",+ "overall_status": "failed",+ "blocks": [{"label": "search_registry", "status": "failed", "failure_reason": "Button missing"}],+ },+ },+ )+ ask_ctx.composition_page_evidence = {+ "workflow_run_id": "wr_ask",+ "observed_after_workflow_run": True,+ "source_tool": "inspect_page_for_composition",+ "current_url": "https://example.test/search",+ "forms": [{"label": "Search", "selector": "#search"}],+ }+ ask_result = {"ok": False, "data": {"workflow_run_id": "wr_ask", "overall_status": "failed"}}++ inject_runtime_authoring_repair_context(ask_ctx, ask_result)++ assert "authoring_repair_context" not in ask_result["data"]+ assert ask_ctx.last_code_authoring_repair_context is None++ stop_ctx = _ctx()+ stop_ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER+ stop_ctx.last_test_non_retriable_nav_error = "net::ERR_NAME_NOT_RESOLVED"+ run_execution_module._record_run_blocks_result(+ stop_ctx,+ {+ "ok": False,+ "data": {+ "workflow_run_id": "wr_stop",+ "overall_status": "failed",+ "blocks": [{"label": "open", "status": "failed", "failure_reason": "net::ERR_NAME_NOT_RESOLVED"}],+ },+ },+ )+ stop_ctx.composition_page_evidence = {+ "workflow_run_id": "wr_stop",+ "observed_after_workflow_run": True,+ "source_tool": "inspect_page_for_composition",+ "current_url": "https://example.test/search",+ "forms": [{"label": "Search", "selector": "#search"}],+ }+ stop_result = {+ "ok": False,+ "error": "Failed to navigate to url https://bad.example.",+ "data": {"workflow_run_id": "wr_stop", "overall_status": "failed"},+ }++ inject_runtime_authoring_repair_context(stop_ctx, stop_result)++ assert "authoring_repair_context" not in stop_result["data"]+ assert stop_ctx.last_code_authoring_repair_context is None+++def test_direct_runtime_authoring_repair_finalization_suppresses_stop_class_state() -> None:+ cases = [+ {+ "failure_reason": "Failed to navigate to url https://bad.example.",+ "ctx_attr": ("last_test_non_retriable_nav_error", "net::ERR_NAME_NOT_RESOLVED"),+ },+ {+ "failure_reason": "Browser session not found while taking screenshot.",+ "ctx_attr": None,+ },+ {+ "failure_reason": "Skipped test run: required credentials are not configured.",+ "ctx_attr": None,+ },+ ]+ for case in cases:+ ctx = _ctx()+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER+ run_execution_module._record_run_blocks_result(+ ctx,+ {+ "ok": False,+ "data": {+ "workflow_run_id": "wr_stop",+ "overall_status": "failed",+ "blocks": [+ {+ "label": "search_registry",+ "status": "failed",+ "failure_reason": case["failure_reason"],+ }+ ],+ },+ },+ )+ ctx_attr = case["ctx_attr"]+ if ctx_attr is not None:+ setattr(ctx, ctx_attr[0], ctx_attr[1])+ ctx.composition_page_evidence = {+ "workflow_run_id": "wr_stop",+ "observed_after_workflow_run": True,+ "source_tool": "inspect_page_for_composition",+ "current_url": "https://example.test/search",+ "forms": [{"label": "Search", "selector": "#search"}],+ }++ assert finalize_runtime_authoring_repair_context_from_page_observation(ctx) is None+ assert ctx.pending_code_authoring_runtime_repair_context is None+ assert ctx.last_code_authoring_repair_context is None+++def test_new_pending_runtime_failure_clears_prior_finalized_runtime_context() -> None:+ ctx = _ctx()+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER+ ctx.last_code_authoring_repair_context = CodeAuthoringRepairContext(+ block_label="old_search",+ reason_code="runtime_block_failure",+ runtime_failure_reason="Old failure",+ workflow_run_id="wr_old",+ current_origin="https://old.example",+ observed_after_workflow_run=True,+ page_form_summaries=["Old #search"],+ )++ run_execution_module._record_run_blocks_result(+ ctx,+ {+ "ok": False,+ "data": {+ "workflow_run_id": "wr_new",+ "overall_status": "failed",+ "blocks": [{"label": "new_search", "status": "failed", "failure_reason": "New button missing"}],+ },+ },+ )++ pending_context = ctx.pending_code_authoring_runtime_repair_context+ assert isinstance(pending_context, CodeAuthoringRepairContext)+ assert pending_context.block_label == "new_search"+ assert pending_context.workflow_run_id == "wr_new"+ assert ctx.last_code_authoring_repair_context is None+++def test_runtime_authoring_repair_context_sanitizes_failure_and_page_summaries() -> None:+ ctx = _ctx()+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER+ run_execution_module._record_run_blocks_result(+ ctx,+ {+ "ok": False,+ "data": {+ "workflow_run_id": "wr_secret",+ "overall_status": "failed",+ "blocks": [+ {+ "label": "search_registry",+ "status": "failed",+ "failure_reason": "Timeout after entering password=hunter2",+ }+ ],+ },+ },+ )+ ctx.composition_page_evidence = {+ "workflow_run_id": "wr_secret",+ "observed_after_workflow_run": True,+ "source_tool": "inspect_page_for_composition",+ "current_url": "https://user:[email protected]/search?password=hunter2",+ "page_title": "Search",+ "forms": [{"label": "Password password=hunter2", "selector": "#password"}],+ "result_containers": [{"label": "Result", "text": "token=secret-token"}],+ }+ result = {"ok": False, "data": {"workflow_run_id": "wr_secret", "overall_status": "failed"}}++ inject_runtime_authoring_repair_context(ctx, result)++ repair_context = ctx.last_code_authoring_repair_context+ assert isinstance(repair_context, CodeAuthoringRepairContext)+ dumped = repair_context.model_dump_json()+ assert "hunter2" not in dumped+ assert "secret-token" not in dumped+ assert "user:secret" not in dumped+ assert "password=hunter2" not in dumped+ assert repair_context.current_origin == "https://example.test"++def test_schema_incompatibility_failure_type_stops_without_repair() -> None:# SKY-11380: the typed schema-incompatibility reject must route to STOP, never repair,# so the agent reports the mismatch instead of churning toward repair_ceiling_reached.diff --git a/tests/unit/test_copilot_evaluate_page_evidence.py b/tests/unit/test_copilot_evaluate_page_evidence.pyindex ea1dfde87..d1da0f987 100644--- a/tests/unit/test_copilot_evaluate_page_evidence.py+++ b/tests/unit/test_copilot_evaluate_page_evidence.py@@ -4,7 +4,9 @@ from unittest.mock import MagicMockimport pytest+from skyvern.forge.sdk.copilot import agent as agent_modulefrom skyvern.forge.sdk.copilot.build_phase import BuildPhase+from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicyfrom skyvern.forge.sdk.copilot.context import CopilotContextfrom skyvern.forge.sdk.copilot.request_policy import RequestPolicyfrom skyvern.forge.sdk.copilot.result_evidence import LoadedResultCompositionEvidence@@ -13,6 +15,7 @@ from skyvern.forge.sdk.copilot.tools import (_inspect_page_for_composition_impl,_mark_pending_browser_interaction_observation,)+from skyvern.forge.sdk.copilot.tools import run_execution as run_execution_modulefrom skyvern.forge.sdk.copilot.tools.blockers import _tool_loop_errorfrom skyvern.forge.sdk.copilot.turn_intent import TurnIntent, TurnIntentAuthority, TurnIntentMode@@ -237,4 +240,79 @@ async def test_target_url_inspection_does_not_navigate_away_from_interaction_evi"observation_step": 4,}assert 'target_url="current_page"' in result["error"]- assert "observation_step 4" in result["error"]+++async def test_current_page_inspection_finalizes_runtime_repair_context_for_next_prompt(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ ctx = _ctx()+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER+ run_execution_module._record_run_blocks_result(+ ctx,+ {+ "ok": False,+ "data": {+ "workflow_run_id": "wr_failed",+ "overall_status": "failed",+ "blocks": [+ {+ "label": "search_registry",+ "status": "failed",+ "failure_reason": 'Timeout waiting for locator("#results")',+ }+ ],+ },+ },+ )++ async def fallback_page_info(_ctx: CopilotContext) -> tuple[str, str]:+ return "https://example.test/search?case=secret", "Search"++ async def capture_evidence(+ _ctx: CopilotContext,+ *,+ inspected_url: str,+ current_url: str,+ ) -> tuple[dict[str, object], None]:+ return (+ {+ "inspected_url": inspected_url,+ "current_url": current_url,+ "page_title": "Search",+ "source_tool": "inspect_page_for_composition",+ "forms": [{"label": "Search", "selector": "#search"}],+ "result_containers": [{"label": "No results", "text": "No matching records"}],+ "navigation_targets": [],+ "challenge_controls": [],+ },+ None,+ )++ async def no_completion_verification(*_args: object, **_kwargs: object) -> None:+ return None++ monkeypatch.setattr(+ "skyvern.forge.sdk.copilot.tools.composition_capture._fallback_page_info",+ fallback_page_info,+ )+ monkeypatch.setattr(+ "skyvern.forge.sdk.copilot.tools.composition_capture._capture_composition_evidence",+ capture_evidence,+ )+ monkeypatch.setattr(+ "skyvern.forge.sdk.copilot.tools.composition_capture._maybe_run_completion_verification_from_page_observation",+ no_completion_verification,+ )++ result = await _inspect_page_for_composition_impl(ctx, "current_page")+ prompt = agent_module._code_authoring_repair_context_prompt(ctx)++ assert result["ok"] is True+ assert ctx.pending_code_authoring_runtime_repair_context is None+ assert ctx.last_code_authoring_repair_context is not None+ assert ctx.last_code_authoring_repair_context.current_origin == "https://example.test"+ assert ctx.last_code_authoring_repair_context.page_result_summaries == ["No results No matching records"]+ assert "runtime_failure_class: timeout_waiting_for_selector" in prompt+ assert "page_results: No results No matching records" in prompt+ assert "case=secret" not in ctx.last_code_authoring_repair_context.model_dump_json()