Skip to content

Automate browser based workflows with AI

SKY-11327 Fix Copilot code authoring repair loop (#6792)

Andrew Neilsonfe1531bd59a8
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 7a3d1a18c..35b9e2fc6 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -44,6 +44,7 @@ from skyvern.forge.sdk.copilot.blocker_signal import (
)
from skyvern.forge.sdk.copilot.blocker_signal import to_trace_data as blocker_signal_to_trace_data
from skyvern.forge.sdk.copilot.build_phase import initial_build_phase
+from skyvern.forge.sdk.copilot.code_block_preflight import SANDBOX_UNRESOLVED_NAME_REASON_CODE
from skyvern.forge.sdk.copilot.code_block_synthesis import (
is_optional_dismissal_only_trajectory,
render_synthesized_offer_text,
@@ -63,6 +64,7 @@ from skyvern.forge.sdk.copilot.config import (
from skyvern.forge.sdk.copilot.context import (
COPILOT_RESPONSE_TYPES,
AgentResult,
+ CodeAuthoringRepairContext,
CopilotContext,
NarrativeActivityEntry,
NarrativeBlock,
@@ -531,6 +533,125 @@ def _runtime_verification_evidence_prompt(ctx: CopilotContext | None) -> str:
)
+def _clean_authoring_repair_prompt_atom(value: str, *, max_chars: int = 160) -> str:
+ cleaned = redact_raw_secrets_for_prompt(value).replace("\r", " ").replace("\n", " ").strip()
+ if contains_internal_machinery_leak(cleaned):
+ return ""
+ return cleaned[:max_chars]
+
+
+def _render_authoring_repair_prompt_list(items: list[str], *, max_items: int = 20) -> str:
+ cleaned = [_clean_authoring_repair_prompt_atom(item) for item in items[:max_items]]
+ return ", ".join(item for item in cleaned if item) or "(none)"
+
+
+def _render_selector_repair_alternatives(alternatives: list[dict[str, str]], *, max_items: int = 8) -> list[str]:
+ lines: list[str] = []
+ for alternative in alternatives[:max_items]:
+ tool_name = _clean_authoring_repair_prompt_atom(str(alternative.get("tool_name") or ""), max_chars=60)
+ role = _clean_authoring_repair_prompt_atom(str(alternative.get("role") or ""), max_chars=80)
+ selector = _clean_authoring_repair_prompt_atom(str(alternative.get("selector") or ""), max_chars=180)
+ if not selector:
+ continue
+ parts = [f"tool_name={tool_name or '(unknown)'}"]
+ if role:
+ parts.append(f"role={role}")
+ parts.append(f"selector={selector}")
+ lines.append("- " + ", ".join(parts))
+ return lines
+
+
+def _render_unresolved_name_binding_actions(
+ unresolved_names: list[str], available_parameter_keys: list[str], *, max_items: int = 20
+) -> list[str]:
+ available_keys = {
+ key
+ for raw_key in available_parameter_keys
+ for key in [_clean_authoring_repair_prompt_atom(raw_key, max_chars=80)]
+ if key
+ }
+ lines: list[str] = []
+ for raw_name in unresolved_names[:max_items]:
+ name = _clean_authoring_repair_prompt_atom(raw_name, max_chars=80)
+ if not name:
+ continue
+ if name in available_keys:
+ lines.append(
+ f"- {name} -> existing workflow parameter key {name} -> parameter_keys -> bare variable {name}"
+ )
+ continue
+ lines.append(
+ f"- {name} -> create workflow string parameter key {name} -> parameter_keys -> bare variable {name}"
+ )
+ return lines
+
+
+def _code_authoring_repair_context_prompt(ctx: CopilotContext | None) -> str:
+ if ctx is None:
+ return ""
+ if normalize_block_authoring_policy(ctx.block_authoring_policy) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
+ return ""
+ repair_context = ctx.last_code_authoring_repair_context
+ if not isinstance(repair_context, CodeAuthoringRepairContext):
+ return ""
+ LOG.info(
+ "copilot code authoring repair context rendered",
+ reason_code=repair_context.reason_code,
+ block_label=repair_context.block_label,
+ unresolved_names=repair_context.unresolved_names,
+ )
+ available_parameter_keys = repair_context.available_parameter_keys
+ binding_candidates = repair_context.binding_candidates or repair_context.unresolved_names
+
+ lines = [
+ "CODE AUTHORING REPAIR CONTEXT:",
+ f"block_label: {_clean_authoring_repair_prompt_atom(repair_context.block_label)}",
+ f"reason_code: {_clean_authoring_repair_prompt_atom(repair_context.reason_code)}",
+ f"unresolved_names: {_render_authoring_repair_prompt_list(repair_context.unresolved_names)}",
+ f"declared_parameter_keys: {_render_authoring_repair_prompt_list(repair_context.parameter_keys)}",
+ f"available_parameter_keys: {_render_authoring_repair_prompt_list(available_parameter_keys)}",
+ f"binding_candidates: {_render_authoring_repair_prompt_list(binding_candidates)}",
+ f"allowed_global_names: {_render_authoring_repair_prompt_list(repair_context.allowed_global_names)}",
+ ]
+ if repair_context.selector:
+ lines.append(f"selector: {_clean_authoring_repair_prompt_atom(repair_context.selector)}")
+ if repair_context.source_url:
+ lines.append(f"source_url: {_clean_authoring_repair_prompt_atom(repair_context.source_url)}")
+ if repair_context.refiner_selector:
+ lines.append(f"refiner_selector: {_clean_authoring_repair_prompt_atom(repair_context.refiner_selector)}")
+ selector_alternative_lines = _render_selector_repair_alternatives(repair_context.selector_alternatives)
+ if selector_alternative_lines:
+ lines.append("same_page_selector_alternatives:")
+ lines.extend(selector_alternative_lines)
+ if repair_context.allowed_helper_surface:
+ lines.append("allowed_helper_surface:")
+ for helper_name, attributes in sorted(repair_context.allowed_helper_surface.items()):
+ helper = _clean_authoring_repair_prompt_atom(helper_name)
+ rendered_attributes = _render_authoring_repair_prompt_list(attributes, max_items=40)
+ if helper:
+ lines.append(f"{helper}: {rendered_attributes}")
+ if repair_context.reason_code == SANDBOX_UNRESOLVED_NAME_REASON_CODE:
+ binding_action_lines = _render_unresolved_name_binding_actions(
+ repair_context.unresolved_names, available_parameter_keys
+ )
+ if binding_action_lines:
+ lines.append("binding_actions:")
+ lines.extend(binding_action_lines)
+ lines.append(
+ "For workflow-input-like unresolved names, ensure a workflow string parameter exists, "
+ "list the exact key in the code block's parameter_keys, reference the exact key as a bare Python "
+ "variable in code, do not hardcode the eval value, and rerun via update_and_run_blocks."
+ )
+ if repair_context.reason_code == "ambiguous_bare_selector":
+ lines.append(
+ "For ambiguous selectors, do not re-emit the bare selector or a positional nth selector. "
+ "Use the same-page alternatives when they are stable, or re-scout the same page and choose a "
+ "stable role/name/data attribute."
+ )
+ lines.append(_clean_authoring_repair_prompt_atom(repair_context.repair_instruction, max_chars=260))
+ return "\n\n" + "\n".join(line for line in lines if line)
+
+
def _synthesized_block_offer_prompt(ctx: CopilotContext | None) -> str:
"""Pre-authoring offer of the synthesized code block.
@@ -610,6 +731,7 @@ def _build_dynamic_system_prompt(tool_usage_guide: str, config: CopilotConfig) -
return (
prompt
+ _runtime_verification_evidence_prompt(ctx)
+ + _code_authoring_repair_context_prompt(ctx)
+ _synthesized_block_offer_prompt(ctx)
+ _docs_answer_turn_directive(ctx.turn_intent)
)
diff --git a/skyvern/forge/sdk/copilot/code_block_preflight.py b/skyvern/forge/sdk/copilot/code_block_preflight.py
index aef6e0272..4dbcaf0ea 100644
--- a/skyvern/forge/sdk/copilot/code_block_preflight.py
+++ b/skyvern/forge/sdk/copilot/code_block_preflight.py
@@ -17,6 +17,8 @@ from typing import Iterable
from skyvern.forge.sdk.copilot.code_block_security import CodeBlockSecurityError, author_time_code_security_errors
from skyvern.forge.sdk.workflow.models.block import CodeBlock
+SANDBOX_UNRESOLVED_NAME_REASON_CODE = "SANDBOX_UNRESOLVED_NAME"
+
@dataclass(frozen=True)
class CodeBlockPreflightDiagnostic:
@@ -24,6 +26,17 @@ class CodeBlockPreflightDiagnostic:
message: str
+@dataclass(frozen=True)
+class CodeBlockSandboxNameDiagnostic:
+ code: str
+ message: str
+ unresolved_names: tuple[str, ...]
+ class_names: tuple[str, ...]
+ parameter_keys: tuple[str, ...]
+ allowed_global_names: tuple[str, ...]
+ allowed_helper_surface: dict[str, tuple[str, ...]]
+
+
_ANSI_ESCAPE_RE = re.compile(r"\x1b\[[0-9;]*m|\x1b\([AB]")
_MYPY_ERROR_RE = re.compile(r"^(?P<path>.*?):(?P<line>\d+): (?P<severity>error): (?P<message>.*)$")
_LOCATOR_NOT_CALLABLE_RE = re.compile(r'"Locator" not callable\s+\[operator\]')
@@ -57,6 +70,14 @@ def _sandbox_shim_surface() -> dict[str, frozenset[str]]:
}
+def sandbox_allowed_global_names() -> list[str]:
+ return sorted(_sandbox_global_names())
+
+
+def sandbox_allowed_helper_surface() -> dict[str, list[str]]:
+ return {name: sorted(surface) for name, surface in sorted(_sandbox_shim_surface().items())}
+
+
def strip_redundant_sandbox_imports(code: str) -> tuple[str, list[str]]:
"""Remove top-level imports the runtime sandbox already injects.
@@ -240,14 +261,26 @@ def sandbox_unresolved_name_diagnostics(
ambiguous control-flow bindings do not satisfy later reads.
"""
+ repair_diagnostic = sandbox_unresolved_name_repair_diagnostic(code, parameter_keys=parameter_keys)
+ if repair_diagnostic is None:
+ return []
+ return [CodeBlockPreflightDiagnostic(code=repair_diagnostic.code, message=repair_diagnostic.message)]
+
+
+def sandbox_unresolved_name_repair_diagnostic(
+ code: str,
+ *,
+ parameter_keys: Iterable[str] = (),
+) -> CodeBlockSandboxNameDiagnostic | None:
try:
tree = ast.parse(textwrap.dedent(code).strip() or "pass")
except SyntaxError:
- return []
+ return None
- unresolved_names, class_names = _SandboxNameAnalyzer(parameter_keys=parameter_keys).analyze(tree.body)
+ parameter_key_list = sorted(key for key in dict.fromkeys(parameter_keys) if _valid_python_identifier(key))
+ unresolved_names, class_names = _SandboxNameAnalyzer(parameter_keys=parameter_key_list).analyze(tree.body)
if not unresolved_names and not class_names:
- return []
+ return None
names = sorted(unresolved_names)
rejected_classes = sorted(class_names)
@@ -259,17 +292,22 @@ def sandbox_unresolved_name_diagnostics(
"class definitions unavailable in the code sandbox: " + ", ".join(f"`{name}`" for name in rejected_classes)
)
detail = "; ".join(detail_parts)
- return [
- CodeBlockPreflightDiagnostic(
- code="SANDBOX_UNRESOLVED_NAME",
- message=(
- f"Code block references names that are unavailable in the runtime code sandbox or are not "
- f"definitely initialized before use ({detail}). The sandbox provides `page`, declared code-block "
- "parameter keys, and its explicit safe helper namespace; `Exception` is the only available "
- "exception type."
- ),
- )
- ]
+ return CodeBlockSandboxNameDiagnostic(
+ code=SANDBOX_UNRESOLVED_NAME_REASON_CODE,
+ message=(
+ f"Code block references names that are unavailable in the runtime code sandbox or are not "
+ f"definitely initialized before use ({detail}). The sandbox provides `page`, declared code-block "
+ "parameter keys, and its explicit safe helper namespace; `Exception` is the only available "
+ "exception type."
+ ),
+ unresolved_names=tuple(names),
+ class_names=tuple(rejected_classes),
+ parameter_keys=tuple(parameter_key_list),
+ allowed_global_names=tuple(sandbox_allowed_global_names()),
+ allowed_helper_surface={
+ helper: tuple(attributes) for helper, attributes in sandbox_allowed_helper_surface().items()
+ },
+ )
def author_time_code_block_diagnostics(code: str) -> list[CodeBlockPreflightDiagnostic]:
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index 06f30869d..9b600d355 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -133,6 +133,22 @@ class ObservedPage(BaseModel):
reached_via: str = ""
+class CodeAuthoringRepairContext(BaseModel):
+ block_label: str
+ reason_code: str
+ unresolved_names: list[str] = Field(default_factory=list)
+ parameter_keys: list[str] = Field(default_factory=list)
+ available_parameter_keys: list[str] = Field(default_factory=list)
+ binding_candidates: list[str] = Field(default_factory=list)
+ selector: str | None = None
+ source_url: str | None = None
+ refiner_selector: str | None = None
+ selector_alternatives: list[dict[str, str]] = Field(default_factory=list)
+ allowed_global_names: list[str] = Field(default_factory=list)
+ allowed_helper_surface: dict[str, list[str]] = Field(default_factory=dict)
+ repair_instruction: str = "add workflow-input-like names to parameter_keys, or stop referencing them."
+
+
class StructuredContext(BaseModel):
user_goal: str = ""
urls_visited: list[UrlVisit] = Field(default_factory=list)
@@ -480,6 +496,7 @@ class CopilotContext(AgentContext):
# True when the most-recent such rejection deferred to the credential-scout
# gate, so the churn backstop yields to that message instead of pre-empting it.
last_code_authoring_reject_was_credential_priority: bool = False
+ last_code_authoring_repair_context: CodeAuthoringRepairContext | None = None
# Turn-scoped monotonic marks of verified forward progress: the union of
# completion criteria the judge confirmed satisfied so far this turn, and the
# high-water length of the verified block prefix. A repair that grows either
diff --git a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
index 688198c4c..ffa79d7bf 100644
--- a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
+++ b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
@@ -1,11 +1,15 @@
from __future__ import annotations
+import hashlib
+import json
+import re
from enum import StrEnum
from typing import TYPE_CHECKING, Any
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict, Field, ValidationError
from skyvern.forge.sdk.copilot.composition_evidence import interactive_challenge_controls
+from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext
from skyvern.forge.sdk.copilot.failure_tracking import (
ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,
RepairRootCauseIdentity,
@@ -29,6 +33,8 @@ _CREDENTIAL_INPUT_MISSING_SKIP_REASONS = {"workflow_credential_inputs_unbound",
_PRE_RUN_CREDENTIAL_FAILURE_CATEGORIES = {"CREDENTIAL_ERROR", "PARAMETER_BINDING_ERROR"}
_REPAIRABLE_RUNTIME_CATEGORIES = {"AUTH_FAILURE", "OUTCOME_UNVERIFIED"}
_TERMINAL_ANTI_BOT_TERMS = ("captcha", "challenge", "verification", "anti-bot", "anti bot", "turnstile")
+_AUTHORING_REPAIR_SIGNATURE_VERSION = "authoring_repair_context:v1"
+_AUTHORING_REPAIR_CATEGORY = "CODE_AUTHORING_REPAIR"
class StrictModel(BaseModel):
@@ -154,7 +160,8 @@ def build_diagnosis_repair_contract(
run_status = _safe_str(data.get("overall_status"))
workflow_run_id = _safe_str(data.get("workflow_run_id"))
summary = _failure_summary(result, data, blocks)
- root_cause_identity = compute_repair_root_cause_signature(
+ repair_context = _current_code_authoring_repair_context(data)
+ root_cause_identity = _repair_context_root_cause_identity(repair_context) or compute_repair_root_cause_signature(
failure_categories=categories,
failure_reason=_safe_str(data.get("failure_reason")),
error_texts=[_safe_str(result.get("error"))],
@@ -174,10 +181,13 @@ def build_diagnosis_repair_contract(
terminal_challenge_categories,
result,
data,
+ repair_context,
)
- next_action = _next_action(failure_type, ctx, data)
+ next_action = _next_action(failure_type, ctx, data, repair_context)
frontier = _safe_str(data.get("frontier_start_label"))
target_blocks = failed_blocks or ([frontier] if frontier else []) if next_action == RepairNextAction.REPAIR else []
+ if next_action == RepairNextAction.REPAIR and not target_blocks and repair_context is not None:
+ target_blocks = [repair_context.block_label]
user_goal_satisfied, completion_contract_satisfied = _verification_satisfaction(
run_ok,
suspicious,
@@ -303,6 +313,50 @@ def _safe_text(value: str | None, max_chars: int = _TEXT_MAX) -> str:
return text if len(text) <= max_chars else text[: max_chars - 3].rstrip() + "..."
+def _safe_identity_list(values: list[str]) -> list[str]:
+ return sorted(dict.fromkeys(item for value in values for item in [_safe_text(str(value), 80)] if item))
+
+
+def _identity_token(value: str) -> str:
+ return re.sub(r"[^a-zA-Z0-9]+", "_", value.strip().lower()).strip("_") or "unknown"
+
+
+def _repair_context_root_cause_identity(
+ repair_context: CodeAuthoringRepairContext | None,
+) -> RepairRootCauseIdentity | None:
+ if repair_context is None:
+ return None
+ reason_code = _safe_text(repair_context.reason_code, 80)
+ if not reason_code:
+ return None
+ block_label = _safe_text(repair_context.block_label, 80)
+ selector = ""
+ selector_kind = ""
+ payload: dict[str, Any] = {
+ "version": _AUTHORING_REPAIR_SIGNATURE_VERSION,
+ "reason_code": reason_code,
+ "block_label": block_label,
+ }
+ if reason_code == "ambiguous_bare_selector":
+ selector = _safe_text(repair_context.selector, 120)
+ refiner_selector = _safe_text(repair_context.refiner_selector, 120)
+ payload["selector"] = selector
+ payload["refiner_selector"] = refiner_selector
+ selector_kind = "selector" if selector else ""
+ elif repair_context.unresolved_names:
+ payload["unresolved_names"] = _safe_identity_list(repair_context.unresolved_names)
+
+ serialized = json.dumps(payload, sort_keys=True, separators=(",", ":"))
+ return RepairRootCauseIdentity(
+ root_cause_signature=hashlib.sha256(serialized.encode("utf-8")).hexdigest(),
+ primary_category=_AUTHORING_REPAIR_CATEGORY,
+ failure_categories=(_AUTHORING_REPAIR_CATEGORY,),
+ error_class=f"code_authoring_{_identity_token(reason_code)}",
+ selector_kind=selector_kind,
+ selector=selector,
+ )
+
+
def _str_list(value: Any) -> list[str]:
return (
[_safe_text(str(item), 80) for item in value[:_MAX_ITEMS] if str(item).strip()]
@@ -391,6 +445,7 @@ def _failure_type(
terminal_challenge_categories: list[str],
result: dict[str, Any],
data: dict[str, Any],
+ repair_context: CodeAuthoringRepairContext | None,
) -> DiagnosisFailureType:
skip_reason = _safe_str(data.get("skip_reason"))
if skip_reason in _CREDENTIAL_INPUT_MISSING_SKIP_REASONS:
@@ -425,6 +480,10 @@ def _failure_type(
or "browser session" in error_text
):
return DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT
+ if _safe_str(data.get("failure_type")) == "missing_credential_or_init":
+ return DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT
+ if repair_context is not None:
+ return DiagnosisFailureType.REPAIRABLE_BLOCK_FAILURE
if outcome_verified:
return DiagnosisFailureType.NO_FAILURE
if failed_blocks or category_set & _REPAIRABLE_RUNTIME_CATEGORIES:
@@ -438,14 +497,14 @@ def _failure_type(
return DiagnosisFailureType.FAILED_RUN if result.get("ok") is False else DiagnosisFailureType.UNKNOWN
-def _next_action(failure_type: DiagnosisFailureType, ctx: CopilotContext, data: dict[str, Any]) -> RepairNextAction:
+def _next_action(
+ failure_type: DiagnosisFailureType,
+ ctx: CopilotContext,
+ data: dict[str, Any],
+ repair_context: CodeAuthoringRepairContext | None,
+) -> RepairNextAction:
if failure_type == DiagnosisFailureType.NO_FAILURE:
return RepairNextAction.NO_CHANGE
- if (
- data.get("skip_reason") == "workflow_credential_inputs_unbound"
- or failure_type == DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT
- ):
- return RepairNextAction.ASK
if failure_type == DiagnosisFailureType.UNRECOVERABLE_TOOL_ERROR:
return RepairNextAction.STOP
if failure_type == DiagnosisFailureType.ACTIVE_RUN_TERMINAL_EVIDENCE:
@@ -460,8 +519,16 @@ def _next_action(failure_type: DiagnosisFailureType, ctx: CopilotContext, data:
DiagnosisFailureType.SUSPICIOUS_SUCCESS,
}:
return RepairNextAction.STOP
+ # The judge confirmed every criterion; no repair path should overwrite an already-verified terminal proposal.
if outcome_fully_verified(ctx):
return RepairNextAction.NO_CHANGE
+ if repair_context is not None:
+ return RepairNextAction.REPAIR
+ if (
+ data.get("skip_reason") == "workflow_credential_inputs_unbound"
+ or failure_type == DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT
+ ):
+ return RepairNextAction.ASK
authority = getattr(getattr(ctx, "turn_intent", None), "authority", None)
if getattr(authority, "requires_user_input", False) or getattr(authority, "may_update_workflow", True) is False:
return RepairNextAction.ASK
@@ -474,6 +541,16 @@ def _next_action(failure_type: DiagnosisFailureType, ctx: CopilotContext, data:
return RepairNextAction.ESCALATE
+def _current_code_authoring_repair_context(data: dict[str, Any]) -> CodeAuthoringRepairContext | None:
+ raw_context = data.get("authoring_repair_context")
+ if not isinstance(raw_context, dict):
+ return None
+ try:
+ return CodeAuthoringRepairContext.model_validate(raw_context)
+ except ValidationError:
+ return None
+
+
def _last_test_anti_bot_is_terminal(ctx: CopilotContext, data: dict[str, Any]) -> bool:
anti_bot_reason = getattr(ctx, "last_test_anti_bot", None)
if not isinstance(anti_bot_reason, str) or not anti_bot_reason.strip():
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index c74f62752..b2569f8f5 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -36,6 +36,7 @@ from skyvern.library.skyvern_browser import SkyvernBrowser
if TYPE_CHECKING:
from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult
+ from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext
from skyvern.forge.sdk.copilot.reached_download_target import ReachedDownloadTarget
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
from skyvern.forge.sdk.copilot.result_evidence import LoadedResultCompositionEvidence
@@ -202,6 +203,7 @@ class AgentContext:
repeated_failure_nudge_emitted_at_streak: int = 0
code_authoring_guardrail_reject_count: int = 0
last_code_authoring_reject_was_credential_priority: bool = False
+ last_code_authoring_repair_context: CodeAuthoringRepairContext | None = None
challenge_gated_proxy_retry_count: int = 0
last_test_non_retriable_nav_error: str | None = None
non_retriable_nav_error_last_emitted_signature: str | None = None
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index f961c6311..c839d416b 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -39,6 +39,7 @@ from skyvern.forge.sdk.copilot.completion_verification import (
from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
+ _AUTHORING_REPAIR_CATEGORY,
DiagnosisRepairContract,
RepairLoopState,
RepairNextAction,
@@ -2349,12 +2350,13 @@ async def _verify_and_record_run_blocks_result(
def _repair_non_convergence_signature(contract: DiagnosisRepairContract) -> str | None:
- """A constant streak token for any REPAIR verdict, ``None`` otherwise. The trigger
- is non-convergence (REPAIR with no verified progress), so the only thing the signature
- must do is tell a contiguous repair streak from a reset on a non-REPAIR verdict."""
if contract.repair_decision.next_action is not RepairNextAction.REPAIR:
return None
- # Constant by design: the streak only distinguishes a REPAIR run from a reset, so the failure specifics are ignored.
+ identity = contract.diagnosis_result.root_cause_identity
+ if identity.primary_category == _AUTHORING_REPAIR_CATEGORY and identity.root_cause_signature:
+ return identity.root_cause_signature
+ if _AUTHORING_REPAIR_CATEGORY in identity.failure_categories and identity.root_cause_signature:
+ return identity.root_cause_signature
return "repair_no_verified_progress"
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index dd4583ae3..8c2e62778 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -11,6 +11,7 @@ from collections.abc import Iterator, Mapping
from dataclasses import dataclass
from dataclasses import field as dataclass_field
from typing import Annotated, Any, Literal, NamedTuple
+from urllib.parse import urlsplit
import structlog
import yaml
@@ -26,8 +27,10 @@ from skyvern.forge.sdk.copilot.blocker_signal import (
stash_blocker_signal,
)
from skyvern.forge.sdk.copilot.code_block_preflight import (
+ SANDBOX_UNRESOLVED_NAME_REASON_CODE,
author_time_code_block_diagnostics,
sandbox_unresolved_name_diagnostics,
+ sandbox_unresolved_name_repair_diagnostic,
strip_redundant_sandbox_imports,
)
from skyvern.forge.sdk.copilot.code_block_security import CodeBlockSecurityError, author_time_code_security_errors
@@ -49,8 +52,8 @@ from skyvern.forge.sdk.copilot.composition_evidence import (
normalize_block_observation_refs,
workflow_target_url,
)
-from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
-from skyvern.forge.sdk.copilot.context import CopilotContext
+from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, normalize_block_authoring_policy
+from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext, CopilotContext
from skyvern.forge.sdk.copilot.data_write_defaults import default_data_write_continue_on_failure
from skyvern.forge.sdk.copilot.enforcement import (
MAX_CODE_AUTHORING_GUARDRAIL_REJECTS,
@@ -67,6 +70,7 @@ from skyvern.forge.sdk.copilot.output_policy import (
evaluate_output_policy,
format_output_policy_tool_error,
output_policy_verdict_to_trace_data,
+ url_origin,
)
from skyvern.forge.sdk.copilot.reached_download_target import (
REGISTERED_DOWNLOAD_OUTPUT_KEYS,
@@ -829,6 +833,170 @@ def _generated_code_preflight_reason_code(error: str | CodeBlockSecurityError) -
return reason_code
+def _unresolved_symbol_repair_context_enabled(ctx: AgentContext) -> bool:
+ return normalize_block_authoring_policy(ctx.block_authoring_policy) == BlockAuthoringPolicy.CODE_ONLY_BROWSER
+
+
+def _declared_string_workflow_parameter_keys(parsed: Mapping[str, Any]) -> set[str]:
+ workflow_definition = parsed.get("workflow_definition")
+ if not isinstance(workflow_definition, Mapping):
+ return set()
+ parameters = workflow_definition.get("parameters")
+ if not isinstance(parameters, list):
+ return set()
+ keys: set[str] = set()
+ for parameter in parameters:
+ if not isinstance(parameter, Mapping):
+ continue
+ key = str(parameter.get("key") or "").strip()
+ if not key or _is_credential_parameter(parameter) or is_sensitive_workflow_parameter(dict(parameter)):
+ continue
+ parameter_type = str(parameter.get("parameter_type") or "").lower()
+ workflow_parameter_type = str(parameter.get("workflow_parameter_type") or "").lower()
+ if parameter_type and parameter_type != "workflow":
+ continue
+ if workflow_parameter_type and workflow_parameter_type != "string":
+ continue
+ keys.add(key)
+ return keys
+
+
+def _code_block_available_binding_keys_by_label(workflow_yaml: str | None) -> dict[str, list[str]]:
+ if workflow_yaml is None:
+ return {}
+ parsed = parse_workflow_yaml(workflow_yaml)
+ if not isinstance(parsed, Mapping):
+ return {}
+ available_by_label: dict[str, set[str]] = {}
+
+ def block_output_key(block: Mapping[str, Any]) -> str | None:
+ label = str(block.get("label") or "").strip()
+ return f"{label}_output" if label else None
+
+ def visit_branch(branch: Mapping[str, Any], available_keys: set[str]) -> None:
+ for key in _ORDERED_CHILD_BLOCK_LIST_KEYS:
+ visit_blocks(branch.get(key), set(available_keys))
+ for branch_key in _ORDERED_BRANCH_LIST_KEYS:
+ branches = branch.get(branch_key)
+ if not isinstance(branches, list):
+ continue
+ for nested_branch in branches:
+ if isinstance(nested_branch, Mapping):
+ visit_branch(nested_branch, set(available_keys))
+
+ def visit_blocks(blocks: Any, available_keys: set[str]) -> set[str]:
+ if not isinstance(blocks, list):
+ return available_keys
+ for block in blocks:
+ if not isinstance(block, Mapping):
+ continue
+ label = str(block.get("label") or "").strip()
+ if label and _enum_or_string_name(block.get("block_type")) == BlockType.CODE.value:
+ available_by_label.setdefault(label, set()).update(available_keys)
+ for key in _ORDERED_CHILD_BLOCK_LIST_KEYS:
+ visit_blocks(block.get(key), set(available_keys))
+ for branch_key in _ORDERED_BRANCH_LIST_KEYS:
+ branches = block.get(branch_key)
+ if not isinstance(branches, list):
+ continue
+ for branch in branches:
+ if isinstance(branch, Mapping):
+ visit_branch(branch, set(available_keys))
+ output_key = block_output_key(block)
+ if output_key:
+ available_keys.add(output_key)
+ return available_keys
+
+ workflow_definition = parsed.get("workflow_definition")
+ blocks = workflow_definition.get("blocks") if isinstance(workflow_definition, Mapping) else None
+ visit_blocks(blocks, _declared_string_workflow_parameter_keys(parsed))
+ return {label: sorted(keys) for label, keys in available_by_label.items()}
+
+
+def _code_block_authoring_repair_context(
+ workflow_yaml: str | None,
+ prior_yaml: str | None,
+) -> CodeAuthoringRepairContext | None:
+ prior_blocks = _workflow_yaml_code_blocks_by_label(prior_yaml)
+ available_binding_keys_by_label = _code_block_available_binding_keys_by_label(workflow_yaml)
+ for label, block in _workflow_yaml_code_blocks_by_label(workflow_yaml).items():
+ code = str(block.get("code") or "")
+ if not code.strip():
+ continue
+ parameter_keys = _code_block_parameter_keys(block)
+ prior_block = prior_blocks.get(label)
+ if (
+ prior_block is not None
+ and str(prior_block.get("code") or "") == code
+ and _code_block_parameter_keys(prior_block) == parameter_keys
+ ):
+ continue
+ try:
+ CodeBlock.is_safe_code(code)
+ except (SyntaxError, InsecureCodeDetected):
+ continue
+ if author_time_code_security_errors(label=label, code=code):
+ continue
+ if author_time_code_block_diagnostics(code):
+ continue
+ diagnostic = sandbox_unresolved_name_repair_diagnostic(code, parameter_keys=parameter_keys)
+ if diagnostic is None or not diagnostic.unresolved_names:
+ continue
+ available_parameter_keys = available_binding_keys_by_label.get(label, [])
+ binding_candidates = list(dict.fromkeys([*available_parameter_keys, *diagnostic.unresolved_names]))
+ return CodeAuthoringRepairContext(
+ block_label=label,
+ reason_code=diagnostic.code,
+ unresolved_names=list(diagnostic.unresolved_names),
+ parameter_keys=list(diagnostic.parameter_keys),
+ available_parameter_keys=available_parameter_keys,
+ binding_candidates=binding_candidates,
+ allowed_global_names=list(diagnostic.allowed_global_names),
+ allowed_helper_surface={
+ helper: list(attributes) for helper, attributes in diagnostic.allowed_helper_surface.items()
+ },
+ repair_instruction=(
+ "For each workflow-input-like unresolved name, create a workflow string parameter with the exact "
+ "same key when no declared parameter exists, add that exact key to this code block's "
+ "parameter_keys, reference it as a bare Python variable in code, do not hardcode the eval value, "
+ "and rerun via update_and_run_blocks."
+ ),
+ )
+ return None
+
+
+def _repair_context_log_values(values: list[str], *, max_items: int = 20) -> list[str]:
+ cleaned: list[str] = []
+ for raw_value in values[:max_items]:
+ value = str(raw_value or "").replace("\r", " ").replace("\n", " ").strip()
+ if not value or _SECRET_LIKE_LITERAL_RE.search(value):
+ continue
+ cleaned.append(value[:80])
+ return cleaned
+
+
+def _set_code_authoring_repair_context(ctx: AgentContext, repair_context: CodeAuthoringRepairContext | None) -> None:
+ ctx.last_code_authoring_repair_context = repair_context
+ if repair_context is not None:
+ LOG.info(
+ "copilot code authoring repair context stored",
+ reason_code=repair_context.reason_code,
+ block_label=repair_context.block_label,
+ unresolved_names=_repair_context_log_values(repair_context.unresolved_names),
+ parameter_keys=_repair_context_log_values(repair_context.parameter_keys),
+ available_parameter_keys=_repair_context_log_values(repair_context.available_parameter_keys),
+ binding_candidates=_repair_context_log_values(repair_context.binding_candidates),
+ )
+
+
+def _clear_code_authoring_repair_context(ctx: AgentContext) -> None:
+ ctx.last_code_authoring_repair_context = None
+
+
+def _is_unresolved_symbol_repair_context(repair_context: CodeAuthoringRepairContext | None) -> bool:
+ return repair_context is not None and repair_context.reason_code == SANDBOX_UNRESOLVED_NAME_REASON_CODE
+
+
_CHURN_REASON_CODES = frozenset({"code_authoring_guardrail_churn", "credential_priority_authoring_churn"})
@@ -1015,12 +1183,17 @@ def _code_block_parameter_contract_error(workflow_yaml: str) -> str | None:
return "\n".join(errors) if errors else None
-def _code_repair_progress_data() -> dict[str, str]:
+def _code_repair_progress_data(
+ repair_context: CodeAuthoringRepairContext | None = None,
+) -> dict[str, Any]:
"""Tag a code-authoring reject so the streaming adapter renders it as quiet de-duplicated progress."""
- return {
+ data: dict[str, Any] = {
"surface_kind": CODE_REPAIR_PROGRESS_SURFACE_KIND,
"progress_text": CODE_REPAIR_PROGRESS_TEXT,
}
+ if repair_context is not None:
+ data["authoring_repair_context"] = repair_context.model_dump(mode="json")
+ return data
def _code_seam_rejection_user_summary(*, metadata_rejected: bool, code_rejected: bool) -> str:
@@ -1036,6 +1209,7 @@ class _SynthesizedCodeImpositionResult:
workflow_yaml: str
substitutions: dict[str, Any] | None = None
violations: list[str] = dataclass_field(default_factory=list)
+ repair_context: CodeAuthoringRepairContext | None = None
_SUBMITTED_LITERAL_METHODS = frozenset({"fill", "type"})
@@ -1049,6 +1223,104 @@ def _compiled_authoring_user_summary() -> str:
return "I need to bind the compiled browser-step code safely before saving this workflow."
+def _ambiguous_bare_selector_repair_context(
+ *,
+ code_block: Mapping[str, Any],
+ dropped: Mapping[str, Any],
+ scout_trajectory: list[ScoutedInteraction],
+) -> CodeAuthoringRepairContext | None:
+ if dropped.get("reason_code") != "ambiguous_bare_selector":
+ return None
+ dropped_selector = str(dropped.get("selector") or "").strip()
+ if not dropped_selector:
+ return None
+ dropped_index = dropped.get("trajectory_index")
+ source_url = ""
+ refiner_selector: str | None = None
+ selector_alternatives: list[dict[str, str]] = []
+ if isinstance(dropped_index, int) and 0 <= dropped_index < len(scout_trajectory):
+ source_url = str(scout_trajectory[dropped_index].get("source_url") or "").strip()
+ for later in scout_trajectory[dropped_index + 1 :]:
+ if source_url and str(later.get("source_url") or "").strip() != source_url:
+ continue
+ later_selector = str(later.get("selector") or "").strip()
+ if later_selector and _selector_refines(dropped_selector, later_selector):
+ refiner_selector = later_selector
+ break
+ if refiner_selector is None:
+ selector_alternatives = _ambiguous_selector_same_page_alternatives(
+ dropped_selector=dropped_selector,
+ source_url=source_url,
+ scout_trajectory=scout_trajectory,
+ dropped_index=dropped_index,
+ )
+ safe_source_url = url_origin(source_url) if source_url else None
+ return CodeAuthoringRepairContext(
+ block_label=str(code_block.get("label") or ""),
+ reason_code="ambiguous_bare_selector",
+ selector=dropped_selector,
+ source_url=safe_source_url,
+ refiner_selector=refiner_selector,
+ selector_alternatives=selector_alternatives,
+ repair_instruction=(
+ "Replace the ambiguous bare selector with a unique selector or role/name locator from the same page."
+ ),
+ )
+
+
+def _same_page_url(left: str, right: str) -> bool:
+ if not left or not right:
+ return False
+ left_parts = urlsplit(left)
+ right_parts = urlsplit(right)
+ return (
+ left_parts.scheme,
+ left_parts.netloc,
+ left_parts.path.rstrip("/"),
+ ) == (
+ right_parts.scheme,
+ right_parts.netloc,
+ right_parts.path.rstrip("/"),
+ )
+
+
+def _safe_selector_repair_atom(value: Any, *, max_chars: int = 160) -> str:
+ text = str(value or "").replace("\r", " ").replace("\n", " ").strip()
+ if not text or len(text) > max_chars:
+ return ""
+ if _SECRET_LIKE_LITERAL_RE.search(text):
+ return ""
+ return text
+
+
+def _ambiguous_selector_same_page_alternatives(
+ *,
+ dropped_selector: str,
+ source_url: str,
+ scout_trajectory: list[ScoutedInteraction],
+ dropped_index: int,
+) -> list[dict[str, str]]:
+ alternatives: list[dict[str, str]] = []
+ seen: set[tuple[str, str, str]] = set()
+ for index, interaction in enumerate(scout_trajectory):
+ if index == dropped_index:
+ continue
+ if not _same_page_url(source_url, str(interaction.get("source_url") or "").strip()):
+ continue
+ tool_name = _safe_selector_repair_atom(interaction.get("tool_name"), max_chars=40)
+ selector = _safe_selector_repair_atom(interaction.get("selector"))
+ if not selector or selector == dropped_selector or _is_positional_selector(selector):
+ continue
+ role = _safe_selector_repair_atom(interaction.get("role"), max_chars=60)
+ alternative = {"tool_name": tool_name, "role": role, "selector": selector}
+ key = (alternative["tool_name"], alternative["role"], alternative["selector"])
+ if key in seen:
+ continue
+ seen.add(key)
+ alternatives.append(alternative)
+ return alternatives
+
+
def _prior_yaml_source(ctx: AgentContext) -> tuple[str, str | None]:
last_yaml = getattr(ctx, "last_workflow_yaml", None)
if isinstance(last_yaml, str) and last_yaml:
@@ -1223,6 +1495,9 @@ _FILTERING_PSEUDO_CLASSES = (
":text(",
":is(",
)
+_EXACT_TEXT_XPATH_TAG_RE = re.compile(
+ r"""^(?:xpath=)?//(?P<tag>[a-zA-Z][a-zA-Z0-9-]*)\s*\[\s*normalize-space\(\s*(?:\.|text\(\))?\s*\)\s*=\s*(?P<quote>['"])[^'"]+(?P=quote)\s*\]\s*$"""
+)
def _qualifier_narrows_to_identity(qualifier: str) -> bool:
@@ -1275,6 +1550,25 @@ def _selector_refines(bare: str, candidate: str) -> bool:
return _qualifier_narrows_to_identity(candidate[len(bare) :])
+def _stable_same_kind_bare_click_refiner(bare: str, candidate: str) -> bool:
+ bare = bare.strip()
+ candidate = candidate.strip()
+ if not bare or not candidate or bare == candidate or _is_positional_selector(candidate):
+ return False
+ if _selector_refines(bare, candidate):
+ return True
+ if bare != "button":
+ return False
+
+ candidate_role = _parse_role_name(candidate)
+ if candidate_role is not None:
+ role_name, accessible_name, suffix = candidate_role
+ return role_name == "button" and bool(accessible_name) and not suffix
+
+ xpath_match = _EXACT_TEXT_XPATH_TAG_RE.match(candidate)
+ return xpath_match is not None and xpath_match.group("tag").casefold() == "button"
+
+
def _bare_drop_superseded_on_screen(
dropped: Mapping[str, Any],
scout_trajectory: list[ScoutedInteraction],
@@ -1303,7 +1597,7 @@ def _bare_drop_superseded_on_screen(
if str(later.get("source_url") or "").strip() != source_url:
continue
later_selector = str(later.get("selector") or "").strip()
- if not _selector_refines(dropped_selector, later_selector):
+ if not _stable_same_kind_bare_click_refiner(dropped_selector, later_selector):
continue
claimed_refiner_indices.add(refiner_index)
return True, {
@@ -2307,6 +2601,7 @@ def _maybe_impose_synthesized_code_block(workflow_yaml: str, ctx: AgentContext)
violations.append("Unable to impose synthesized code block: scout trajectory was truncated.")
claimed_refiner_indices: set[int] = set()
forgiven_superseded_bare_drops: list[dict[str, Any]] = []
+ repair_context: CodeAuthoringRepairContext | None = None
for dropped in diagnostics.dropped_interactions:
if _is_ignorable_entry_opener_drop(dropped, diagnostics):
continue
@@ -2323,6 +2618,12 @@ def _maybe_impose_synthesized_code_block(workflow_yaml: str, ctx: AgentContext)
violations.append(
f"Unable to impose synthesized code block: dropped scout interaction {index} from `{tool_name}` ({reason})."
)
+ if repair_context is None:
+ repair_context = _ambiguous_bare_selector_repair_context(
+ code_block=code_block,
+ dropped=dropped,
+ scout_trajectory=scout_trajectory,
+ )
for provenance in diagnostics.locator_provenance:
if not _locator_provenance_is_self_validating(provenance):
violations.append("Unable to impose synthesized code block: locator provenance was not byte-equal.")
@@ -2337,7 +2638,11 @@ def _maybe_impose_synthesized_code_block(workflow_yaml: str, ctx: AgentContext)
)
violations.extend(parameter_reconciliation.violations)
if violations:
- return _SynthesizedCodeImpositionResult(workflow_yaml=workflow_yaml, violations=violations)
+ return _SynthesizedCodeImpositionResult(
+ workflow_yaml=workflow_yaml,
+ violations=violations,
+ repair_context=repair_context,
+ )
extraction_suffix = _submitted_suffix_after_synthesized_code(submitted_code, synthesized.code)
raw_metadata = getattr(ctx, "raw_code_artifact_metadata", None)
@@ -3163,19 +3468,36 @@ async def _update_workflow(
*,
allow_missing_credentials: bool | None = None,
) -> dict[str, Any]:
+ def reject(
+ *,
+ error: str,
+ user_facing_summary: str | None = None,
+ data: dict[str, Any] | None = None,
+ repair_context: CodeAuthoringRepairContext | None = None,
+ ) -> dict[str, Any]:
+ if repair_context is None:
+ _clear_code_authoring_repair_context(ctx)
+ else:
+ _set_code_authoring_repair_context(ctx, repair_context)
+ result: dict[str, Any] = {"ok": False, "error": error}
+ if user_facing_summary is not None:
+ result["user_facing_summary"] = user_facing_summary
+ if data is not None:
+ result["data"] = data
+ return result
+
authority_error = _authority_tool_error(ctx, "update_workflow")
if authority_error is not None:
- return {"ok": False, "error": authority_error}
+ return reject(error=authority_error)
workflow_yaml = params["workflow_yaml"]
raw_conflict_marker_error = _raw_workflow_yaml_conflict_marker_error(workflow_yaml)
if raw_conflict_marker_error is not None:
- return {
- "ok": False,
- "error": raw_conflict_marker_error,
- "user_facing_summary": _compiled_authoring_user_summary(),
- "data": _code_repair_progress_data(),
- }
+ return reject(
+ error=raw_conflict_marker_error,
+ user_facing_summary=_compiled_authoring_user_summary(),
+ data=_code_repair_progress_data(),
+ )
# Tool wrappers run authority/loop guards before calling here. The composition
# gate below consumes these refs, so they must be visible before validation.
ctx.raw_block_observation_refs = params.get("raw_block_observation_refs", params.get("block_observation_refs"))
@@ -3184,45 +3506,42 @@ async def _update_workflow(
# Imposition reconciles synthesized aliases/parameters before the persisted YAML contract is checked.
imposition = _maybe_impose_synthesized_code_block(workflow_yaml, ctx)
if imposition.violations:
- return {
- "ok": False,
- "error": "\n".join(imposition.violations),
- "user_facing_summary": _compiled_authoring_user_summary(),
- "data": _code_repair_progress_data(),
- }
+ return reject(
+ error="\n".join(imposition.violations),
+ user_facing_summary=_compiled_authoring_user_summary(),
+ data=_code_repair_progress_data(imposition.repair_context),
+ repair_context=imposition.repair_context,
+ )
workflow_yaml = imposition.workflow_yaml
stripped_sandbox_imports: list[str] = []
if _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER:
workflow_yaml, stripped_sandbox_imports = _strip_redundant_sandbox_imports_in_yaml(workflow_yaml)
workflow_yaml, typed_default_violations = _apply_scouted_typed_default_promotions(workflow_yaml, ctx)
if typed_default_violations:
- return {
- "ok": False,
- "error": "\n".join(typed_default_violations),
- "user_facing_summary": _compiled_authoring_user_summary(),
- "data": _code_repair_progress_data(),
- }
+ return reject(
+ error="\n".join(typed_default_violations),
+ user_facing_summary=_compiled_authoring_user_summary(),
+ data=_code_repair_progress_data(),
+ )
params["workflow_yaml"] = workflow_yaml
parameter_contract_error = _code_block_parameter_contract_error(workflow_yaml)
if parameter_contract_error is not None:
- return {
- "ok": False,
- "error": parameter_contract_error,
- "user_facing_summary": _compiled_authoring_user_summary(),
- "data": _code_repair_progress_data(),
- }
+ return reject(
+ error=parameter_contract_error,
+ user_facing_summary=_compiled_authoring_user_summary(),
+ data=_code_repair_progress_data(),
+ )
missing_metadata_error = _missing_code_artifact_metadata_error(
workflow_yaml,
ctx,
params.get("code_artifact_metadata"),
)
if missing_metadata_error is not None:
- return {
- "ok": False,
- "error": missing_metadata_error,
- "user_facing_summary": _compiled_authoring_user_summary(),
- "data": _code_repair_progress_data(),
- }
+ return reject(
+ error=missing_metadata_error,
+ user_facing_summary=_compiled_authoring_user_summary(),
+ data=_code_repair_progress_data(),
+ )
scout_trajectory = getattr(ctx, "scout_trajectory", None)
normalization = _normalize_code_artifact_metadata_detailed(
params.get("code_artifact_metadata"),
@@ -3236,13 +3555,22 @@ async def _update_workflow(
record_code_artifact_violations(ctx, normalization.violations, normalization.offending_labels)
prior_workflow_yaml = getattr(ctx, "workflow_yaml", None)
code_safety_errors = _code_block_safety_errors(workflow_yaml, prior_workflow_yaml)
+ code_authoring_repair_context = (
+ _code_block_authoring_repair_context(workflow_yaml, prior_workflow_yaml)
+ if code_safety_errors and _unresolved_symbol_repair_context_enabled(ctx)
+ else None
+ )
credential_scout_errors = (
[]
if _request_policy_allows_untested_code_block_draft(ctx)
else _credentialed_code_block_scout_gate_errors(workflow_yaml, ctx)
)
- credential_priority_reject = bool(credential_scout_errors) and code_artifact_metadata_error is None
+ unresolved_symbol_priority_reject = _is_unresolved_symbol_repair_context(code_authoring_repair_context)
+ credential_priority_reject = (
+ bool(credential_scout_errors) and code_artifact_metadata_error is None and not unresolved_symbol_priority_reject
+ )
if code_safety_errors:
+ _set_code_authoring_repair_context(ctx, code_authoring_repair_context)
_record_code_authoring_guardrail_reject(ctx, defer_churn_stop=credential_priority_reject)
# Per-label salvage keeps conforming metadata across a rejection; a
# rejected code block keeps nothing, since its yaml never becomes the
@@ -3261,16 +3589,36 @@ async def _update_workflow(
ctx.code_artifact_metadata = merged_metadata
ctx.workflow_verification_evidence.code_artifact_metadata = merged_metadata
params["code_artifact_metadata"] = merged_metadata
+ if (
+ credential_scout_errors
+ and code_safety_errors
+ and code_artifact_metadata_error is None
+ and unresolved_symbol_priority_reject
+ ):
+ return reject(
+ error="\n".join(str(error) for error in code_safety_errors if error),
+ user_facing_summary=_code_seam_rejection_user_summary(
+ metadata_rejected=False,
+ code_rejected=True,
+ ),
+ data=_code_repair_progress_data(code_authoring_repair_context),
+ repair_context=code_authoring_repair_context,
+ )
if credential_scout_errors and code_safety_errors and code_artifact_metadata_error is None:
- return {
- "ok": False,
- "error": "\n".join(credential_scout_errors),
- "user_facing_summary": CREDENTIAL_SCOUT_VERIFY_REPLY,
- "data": {
+ return reject(
+ error="\n".join(credential_scout_errors),
+ user_facing_summary=CREDENTIAL_SCOUT_VERIFY_REPLY,
+ data={
"failure_type": "missing_credential_or_init",
"diagnostic_code_safety_errors": code_safety_errors,
+ **(
+ {"authoring_repair_context": code_authoring_repair_context.model_dump(mode="json")}
+ if code_authoring_repair_context is not None
+ else {}
+ ),
},
- }
+ repair_context=code_authoring_repair_context,
+ )
seam_errors = [
error
for error in (
@@ -3280,29 +3628,28 @@ async def _update_workflow(
if error
]
if seam_errors:
- return {
- "ok": False,
- "error": "\n".join(seam_errors),
- "user_facing_summary": _code_seam_rejection_user_summary(
+ return reject(
+ error="\n".join(seam_errors),
+ user_facing_summary=_code_seam_rejection_user_summary(
metadata_rejected=code_artifact_metadata_error is not None,
code_rejected=bool(code_safety_errors),
),
- "data": _code_repair_progress_data(),
- }
+ data=_code_repair_progress_data(code_authoring_repair_context),
+ repair_context=code_authoring_repair_context,
+ )
if credential_scout_errors:
_record_code_authoring_guardrail_reject(ctx, defer_churn_stop=True)
- return {
- "ok": False,
- "error": "\n".join(credential_scout_errors),
- "user_facing_summary": CREDENTIAL_SCOUT_VERIFY_REPLY,
- "data": {"failure_type": "missing_credential_or_init"},
- }
+ return reject(
+ error="\n".join(credential_scout_errors),
+ user_facing_summary=CREDENTIAL_SCOUT_VERIFY_REPLY,
+ data={"failure_type": "missing_credential_or_init"},
+ )
if allow_missing_credentials is None:
allow_missing_credentials = getattr(ctx, "allow_untested_workflow_draft", False) is True
if not allow_missing_credentials:
credential_error = await _credential_reference_validation_error(workflow_yaml, ctx)
if credential_error is not None:
- return {"ok": False, "error": credential_error}
+ return reject(error=credential_error)
misbinding_findings = _credential_id_misbinding_findings(workflow_yaml)
if misbinding_findings:
@@ -3312,7 +3659,7 @@ async def _update_workflow(
workflow_id=ctx.workflow_id,
findings=misbinding_findings,
)
- return {"ok": False, "error": _credential_id_misbinding_error_message(misbinding_findings)}
+ return reject(error=_credential_id_misbinding_error_message(misbinding_findings))
output_policy_verdict = evaluate_output_policy(
request_policy=getattr(ctx, "request_policy", None),
@@ -3329,7 +3676,7 @@ async def _update_workflow(
tool_name="update_workflow",
),
)
- return {"ok": False, "error": format_output_policy_tool_error(output_policy_verdict)}
+ return reject(error=format_output_policy_tool_error(output_policy_verdict))
# Prefer the most-recent in-turn emission so cross-path flows (inline
# REPLACE_WORKFLOW followed by update_workflow) compare against what the
@@ -3338,15 +3685,15 @@ async def _update_workflow(
prior_yaml = last_yaml if isinstance(last_yaml, str) and last_yaml else ctx.workflow_yaml
stale_metadata = _detect_stale_block_metadata(workflow_yaml, prior_yaml)
if stale_metadata:
- return {"ok": False, "error": _stale_block_metadata_message(stale_metadata)}
+ return reject(error=_stale_block_metadata_message(stale_metadata))
wait_block_error = _timing_only_challenge_wait_reject_message(ctx, workflow_yaml)
if wait_block_error:
- return {"ok": False, "error": wait_block_error}
+ return reject(error=wait_block_error)
challenge_http_error = _challenge_http_request_reject_message(ctx, workflow_yaml, ctx.workflow_yaml)
if challenge_http_error:
- return {"ok": False, "error": challenge_http_error}
+ return reject(error=challenge_http_error)
# Post-emission reject of copilot-v2 writes that introduce a banned
# block type. The schema pre_hook only fires when the LLM consults the
@@ -3359,15 +3706,15 @@ async def _update_workflow(
)
if banned_items:
_record_banned_block_reject_span("_update_workflow", banned_items)
- return {"ok": False, "error": _banned_block_reject_message(banned_items, ctx)}
+ return reject(error=_banned_block_reject_message(banned_items, ctx))
download_scout_error = _download_scout_required_error(ctx, workflow_yaml)
if download_scout_error:
- return {"ok": False, "error": download_scout_error}
+ return reject(error=download_scout_error)
download_binding_error = _download_binding_required_error(ctx, workflow_yaml)
if download_binding_error:
- return {"ok": False, "error": download_binding_error}
+ return reject(error=download_binding_error)
composition_evidence_error = composition_page_evidence_error(ctx, workflow_yaml)
if composition_evidence_error:
@@ -3376,7 +3723,7 @@ async def _update_workflow(
workflow_permanent_id=ctx.workflow_permanent_id,
target_url=workflow_target_url(workflow_yaml),
)
- return {"ok": False, "error": composition_evidence_error}
+ return reject(error=composition_evidence_error)
# New data-write blocks default to surfacing failures rather than swallowing them.
workflow_yaml = default_data_write_continue_on_failure(workflow_yaml, ctx.workflow_yaml)
@@ -3445,6 +3792,7 @@ async def _update_workflow(
ctx.workflow_yaml = workflow_yaml
ctx.code_authoring_guardrail_reject_count = 0
ctx.last_code_authoring_reject_was_credential_priority = False
+ _clear_code_authoring_repair_context(ctx)
_clear_held_churn_signals(ctx)
accepted_metadata = getattr(ctx, "code_artifact_metadata", None)
if isinstance(accepted_metadata, dict) and accepted_metadata:
@@ -3477,17 +3825,19 @@ async def _update_workflow(
"_workflow": workflow,
}
except (yaml.YAMLError, ValidationError, BaseWorkflowHTTPException) as e:
- result: dict[str, Any] = {
- "ok": False,
- "error": f"Workflow validation failed: {e}",
- }
+ repair_data = None
+ user_facing_summary = None
if _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER:
- result["data"] = _code_repair_progress_data()
- result["user_facing_summary"] = _code_seam_rejection_user_summary(
+ repair_data = _code_repair_progress_data()
+ user_facing_summary = _code_seam_rejection_user_summary(
metadata_rejected=False,
code_rejected=True,
)
- return result
+ return reject(
+ error=f"Workflow validation failed: {e}",
+ user_facing_summary=user_facing_summary,
+ data=repair_data,
+ )
def _record_workflow_proxy_location_span(workflow_yaml: str, workflow: Workflow) -> None:
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index d329fb231..3d536c35e 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -21,6 +21,7 @@ from skyvern.forge.sdk.copilot.agent import (
)
from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
from skyvern.forge.sdk.copilot.build_phase import BuildPhase
+from skyvern.forge.sdk.copilot.code_block_preflight import SANDBOX_UNRESOLVED_NAME_REASON_CODE
from skyvern.forge.sdk.copilot.completion_criteria_store import (
StoredCriteriaSet,
StoredCriteriaSnapshot,
@@ -30,6 +31,7 @@ from skyvern.forge.sdk.copilot.completion_criteria_store import (
)
from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict
from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, CopilotConfig
+from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext
from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
DiagnosisInput,
DiagnosisRepairContract,
@@ -633,6 +635,145 @@ workflow_definition:
assert "per_tool_budget_on_block:" in prompt
assert "run only missing block labels" in prompt
+ def test_unresolved_symbol_repair_context_prompt_is_policy_gated(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ info_calls: list[tuple[str, dict[str, str | list[str]]]] = []
+
+ def capture_info(event: str, **kwargs: str | list[str]) -> None:
+ info_calls.append((event, kwargs))
+
+ monkeypatch.setattr(agent_module.LOG, "info", capture_info)
+ repair_context = CodeAuthoringRepairContext(
+ block_label="search_registry",
+ reason_code=SANDBOX_UNRESOLVED_NAME_REASON_CODE,
+ unresolved_names=["confirmation_number"],
+ parameter_keys=[],
+ available_parameter_keys=["confirmation_number"],
+ binding_candidates=["confirmation_number"],
+ allowed_global_names=["Exception", "json", "page"],
+ allowed_helper_surface={"json": ["dumps", "loads"]},
+ )
+ enabled_ctx = _ctx(
+ block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER,
+ last_code_authoring_repair_context=repair_context,
+ )
+ standard_ctx = _ctx(
+ block_authoring_policy=BlockAuthoringPolicy.STANDARD,
+ last_code_authoring_repair_context=repair_context,
+ )
+ wrong_reason_ctx = _ctx(
+ block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER,
+ last_code_authoring_repair_context=repair_context.model_copy(
+ update={"reason_code": "SANDBOX_SAFETY_CHECK"}
+ ),
+ )
+
+ enabled_prompt = agent_module._code_authoring_repair_context_prompt(enabled_ctx)
+
+ assert "CODE AUTHORING REPAIR CONTEXT" in enabled_prompt
+ assert "block_label: search_registry" in enabled_prompt
+ assert "unresolved_names: confirmation_number" in enabled_prompt
+ assert "declared_parameter_keys: (none)" in enabled_prompt
+ assert "available_parameter_keys: confirmation_number" in enabled_prompt
+ assert "binding_candidates: confirmation_number" in enabled_prompt
+ assert (
+ "confirmation_number -> existing workflow parameter key confirmation_number -> parameter_keys -> "
+ "bare variable confirmation_number"
+ ) in enabled_prompt
+ assert "ensure a workflow string parameter exists" in enabled_prompt
+ assert "list the exact key in the code block's parameter_keys" in enabled_prompt
+ assert "do not hardcode the eval value" in enabled_prompt
+ assert "rerun via update_and_run_blocks" in enabled_prompt
+ assert "json: dumps, loads" in enabled_prompt
+ assert "create a workflow string parameter" not in enabled_prompt
+ assert agent_module._code_authoring_repair_context_prompt(standard_ctx) == ""
+ wrong_reason_prompt = agent_module._code_authoring_repair_context_prompt(wrong_reason_ctx)
+ assert "CODE AUTHORING REPAIR CONTEXT" in wrong_reason_prompt
+ assert "reason_code: SANDBOX_SAFETY_CHECK" in wrong_reason_prompt
+ assert (
+ "copilot code authoring repair context rendered",
+ {
+ "reason_code": SANDBOX_UNRESOLVED_NAME_REASON_CODE,
+ "block_label": "search_registry",
+ "unresolved_names": ["confirmation_number"],
+ },
+ ) in info_calls
+
+ def test_unresolved_symbol_repair_context_prompt_creates_exact_parameter_when_missing(self) -> None:
+ repair_context = CodeAuthoringRepairContext(
+ block_label="order_status",
+ reason_code=SANDBOX_UNRESOLVED_NAME_REASON_CODE,
+ unresolved_names=["confirmation_number"],
+ parameter_keys=[],
+ available_parameter_keys=[],
+ binding_candidates=["confirmation_number"],
+ )
+ ctx = _ctx(
+ block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER,
+ last_code_authoring_repair_context=repair_context,
+ )
+
+ prompt = agent_module._code_authoring_repair_context_prompt(ctx)
+
+ assert "available_parameter_keys: (none)" in prompt
+ assert "binding_candidates: confirmation_number" in prompt
+ assert (
+ "confirmation_number -> create workflow string parameter key confirmation_number -> parameter_keys -> "
+ "bare variable confirmation_number"
+ ) in prompt
+ assert "do not hardcode the eval value" in prompt
+ assert "rerun via update_and_run_blocks" in prompt
+
+ def test_ambiguous_selector_repair_context_prompt_includes_selector_details(self) -> None:
+ repair_context = CodeAuthoringRepairContext(
+ block_label="order_status",
+ reason_code="ambiguous_bare_selector",
+ selector="button",
+ source_url="https://example.com/orders",
+ refiner_selector='button[data-action="status"]',
+ repair_instruction="Use a unique selector from the same page before saving the code block.",
+ )
+ ctx = _ctx(
+ block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER,
+ last_code_authoring_repair_context=repair_context,
+ )
+
+ prompt = agent_module._code_authoring_repair_context_prompt(ctx)
+
+ assert "CODE AUTHORING REPAIR CONTEXT" in prompt
+ assert "reason_code: ambiguous_bare_selector" in prompt
+ assert "selector: button" in prompt
+ assert "source_url: https://example.com/orders" in prompt
+ assert 'refiner_selector: button[data-action="status"]' in prompt
+ assert "Use a unique selector from the same page" in prompt
+
+ def test_ambiguous_selector_repair_context_prompt_includes_same_page_alternatives(self) -> None:
+ repair_context = CodeAuthoringRepairContext(
+ block_label="order_status",
+ reason_code="ambiguous_bare_selector",
+ selector="button",
+ source_url="https://example.com",
+ refiner_selector=None,
+ selector_alternatives=[
+ {"tool_name": "type_text", "role": "textbox", "selector": "#order-id"},
+ {"tool_name": "click", "role": "button", "selector": 'role=button[name="Order status"]'},
+ ],
+ repair_instruction="Replace the ambiguous bare selector with a stable same-page control.",
+ )
+ ctx = _ctx(
+ block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER,
+ last_code_authoring_repair_context=repair_context,
+ )
+
+ prompt = agent_module._code_authoring_repair_context_prompt(ctx)
+
+ assert "same_page_selector_alternatives:" in prompt
+ assert "tool_name=type_text, role=textbox, selector=#order-id" in prompt
+ assert 'tool_name=click, role=button, selector=role=button[name="Order status"]' in prompt
+ assert "re-scout the same page" in prompt
+ assert "stable role/name/data attribute" in prompt
+ assert "button:nth-of-type" not in prompt
+ assert "secret-token" not in prompt
+
class TestVerifiedWorkflowOrNone:
"""SKY-9143 strict invariant: a proposal surfaces only after a passing test this turn."""
diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.py
index 29b69540f..9a9e718f9 100644
--- a/tests/unit/test_copilot_code_block_persist_seam.py
+++ b/tests/unit/test_copilot_code_block_persist_seam.py
@@ -17,15 +17,20 @@ from skyvern.forge.sdk.copilot.blocker_signal import (
CopilotToolBlockerSignal,
assert_clean_user_facing_text,
)
-from skyvern.forge.sdk.copilot.code_block_preflight import _sandbox_shim_surface, strip_redundant_sandbox_imports
+from skyvern.forge.sdk.copilot.code_block_preflight import (
+ SANDBOX_UNRESOLVED_NAME_REASON_CODE,
+ _sandbox_shim_surface,
+ strip_redundant_sandbox_imports,
+)
from skyvern.forge.sdk.copilot.code_block_synthesis import _get_by_role_expr, _get_by_role_expr_strict
from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
-from skyvern.forge.sdk.copilot.context import CopilotContext
+from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext, CopilotContext
from skyvern.forge.sdk.copilot.enforcement import (
MAX_CODE_AUTHORING_GUARDRAIL_REJECTS,
MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS,
_check_enforcement,
)
+from skyvern.forge.sdk.copilot.output_utils import sanitize_tool_result_for_llm
from skyvern.forge.sdk.copilot.reached_download_target import ReachedDownloadTarget
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
from skyvern.forge.sdk.copilot.run_outcome import TERMINAL_CHALLENGE_BLOCKER_REASON_CODE
@@ -454,6 +459,15 @@ def _terminal_metadata(label: str, declared_goal: str) -> dict:
}
+def _stale_unresolved_repair_context() -> CodeAuthoringRepairContext:
+ return CodeAuthoringRepairContext(
+ block_label="stale_block",
+ reason_code=SANDBOX_UNRESOLVED_NAME_REASON_CODE,
+ unresolved_names=["stale_name"],
+ parameter_keys=[],
+ )
+
+
class TestCodeSafetySeam:
def test_import_in_new_code_block_is_a_seam_error(self) -> None:
errors = _code_block_safety_errors(_IMPORTING_CODE_YAML, None)
@@ -691,6 +705,282 @@ class TestCodeSafetySeam:
assert result["ok"] is False
assert ctx.code_artifact_metadata == {}
+ @pytest.mark.asyncio
+ async def test_code_only_unresolved_name_rejection_records_repair_context_and_accept_clears(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ ctx = _code_only_ctx()
+
+ rejected = await _update_workflow({"workflow_yaml": _code_yaml("print(provider_query)")}, ctx)
+
+ assert rejected["ok"] is False
+ repair_context = ctx.last_code_authoring_repair_context
+ assert isinstance(repair_context, CodeAuthoringRepairContext)
+ result_context = rejected["data"]["authoring_repair_context"]
+ assert result_context["block_label"] == "search_registry"
+ assert result_context["reason_code"] == SANDBOX_UNRESOLVED_NAME_REASON_CODE
+ assert result_context["unresolved_names"] == ["provider_query"]
+ assert result_context["parameter_keys"] == []
+ assert result_context["available_parameter_keys"] == []
+ assert result_context["binding_candidates"] == ["provider_query"]
+ assert "page" in result_context["allowed_global_names"]
+ assert "json" in result_context["allowed_global_names"]
+ assert "dumps" in result_context["allowed_helper_surface"]["json"]
+ assert "print(provider_query)" not in str(result_context)
+ assert (
+ sanitize_tool_result_for_llm("update_workflow", rejected)["data"]["authoring_repair_context"]
+ == result_context
+ )
+
+ _stub_successful_update(monkeypatch)
+ accepted = await _update_workflow({"workflow_yaml": _SAFE_CODE_YAML}, ctx)
+
+ assert accepted["ok"] is True
+ assert ctx.last_code_authoring_repair_context is None
+
+ @pytest.mark.asyncio
+ async def test_unresolved_name_repair_context_includes_existing_workflow_binding_candidates(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ info_calls: list[tuple[str, dict[str, str | list[str]]]] = []
+
+ def capture_info(event: str, **kwargs: str | list[str]) -> None:
+ info_calls.append((event, kwargs))
+
+ monkeypatch.setattr(workflow_update_module.LOG, "info", capture_info)
+ ctx = _code_only_ctx()
+ submitted = _yaml(
+ """
+ title: Registry lookup
+ workflow_definition:
+ parameters:
+ - {parameter_type: workflow, workflow_parameter_type: string, key: provider_query, default_value: Sample}
+ - {parameter_type: workflow, workflow_parameter_type: string, key: search_location, default_value: City}
+ - {parameter_type: workflow, workflow_parameter_type: credential_id, key: login_credential}
+ blocks:
+ - block_type: code
+ label: search_registry
+ code: |
+ print(provider_name)
+ """
+ )
+
+ rejected = await _update_workflow({"workflow_yaml": submitted}, ctx)
+
+ assert rejected["ok"] is False
+ repair_context = rejected["data"]["authoring_repair_context"]
+ assert repair_context["reason_code"] == SANDBOX_UNRESOLVED_NAME_REASON_CODE
+ assert repair_context["unresolved_names"] == ["provider_name"]
+ assert repair_context["parameter_keys"] == []
+ assert repair_context["available_parameter_keys"] == ["provider_query", "search_location"]
+ assert repair_context["binding_candidates"] == ["provider_query", "search_location", "provider_name"]
+ assert "login_credential" not in str(repair_context)
+ assert (
+ "copilot code authoring repair context stored",
+ {
+ "reason_code": SANDBOX_UNRESOLVED_NAME_REASON_CODE,
+ "block_label": "search_registry",
+ "unresolved_names": ["provider_name"],
+ "parameter_keys": [],
+ "available_parameter_keys": ["provider_query", "search_location"],
+ "binding_candidates": ["provider_query", "search_location", "provider_name"],
+ },
+ ) in info_calls
+
+ @pytest.mark.asyncio
+ async def test_standard_policy_unresolved_name_rejection_has_no_repair_context(self) -> None:
+ ctx = _standard_ctx()
+
+ result = await _update_workflow({"workflow_yaml": _code_yaml("print(provider_query)")}, ctx)
+
+ assert result["ok"] is False
+ assert "authoring_repair_context" not in result["data"]
+ assert ctx.last_code_authoring_repair_context is None
+
+ @pytest.mark.parametrize(
+ "workflow_yaml",
+ [
+ _SAFE_CODE_YAML.replace('await page.goto("https://example.com/search")', "await page.goto("),
+ _REQUESTS_IMPORT_CODE_YAML,
+ _code_yaml('await page.evaluate("1 + 1")'),
+ _code_yaml("""await page.wait_for_function("document.body.innerText.includes('Submitted')")"""),
+ ],
+ )
+ @pytest.mark.asyncio
+ async def test_non_name_authoring_rejects_do_not_carry_repair_context(self, workflow_yaml: str) -> None:
+ ctx = _code_only_ctx()
+
+ result = await _update_workflow({"workflow_yaml": workflow_yaml}, ctx)
+
+ assert result["ok"] is False
+ assert "authoring_repair_context" not in result["data"]
+ assert ctx.last_code_authoring_repair_context is None
+
+ @pytest.mark.parametrize(
+ "workflow_yaml",
+ [
+ _code_yaml("import os\nprint(provider_query)"),
+ _code_yaml("await page.evaluate(provider_query)"),
+ ],
+ )
+ @pytest.mark.asyncio
+ async def test_mixed_primary_authoring_rejects_do_not_carry_repair_context(self, workflow_yaml: str) -> None:
+ ctx = _code_only_ctx()
+
+ result = await _update_workflow({"workflow_yaml": workflow_yaml}, ctx)
+
+ assert result["ok"] is False
+ assert "authoring_repair_context" not in result["data"]
+ assert ctx.last_code_authoring_repair_context is None
+
+ @pytest.mark.asyncio
+ async def test_imposition_reject_carries_ambiguous_selector_repair_context(self) -> None:
+ ctx = _resale_ctx(refiner_selector="button:nth-of-type(2)")
+ private_url = "https://example.com/orders?session=secret-token#account"
+ for interaction in ctx.scout_trajectory:
+ interaction["source_url"] = private_url
+
+ result = await _update_workflow({"workflow_yaml": _resale_submitted_yaml("button:nth-of-type(2)")}, ctx)
+
+ assert result["ok"] is False
+ repair_context = result["data"]["authoring_repair_context"]
+ assert repair_context["reason_code"] == "ambiguous_bare_selector"
+ assert repair_context["block_label"] == "order_status"
+ assert repair_context["selector"] == "button"
+ assert repair_context["source_url"] == "https://example.com"
+ assert "secret-token" not in str(repair_context)
+ assert repair_context["refiner_selector"] is None
+ assert repair_context["selector_alternatives"] == [
+ {"tool_name": "type_text", "role": "textbox", "selector": "#order-id"}
+ ]
+ assert isinstance(ctx.last_code_authoring_repair_context, CodeAuthoringRepairContext)
+ assert ctx.last_code_authoring_repair_context.model_dump(mode="json") == repair_context
+
+ def test_ambiguous_selector_repair_context_only_carries_valid_refiner(self) -> None:
+ code_block = {"label": "order_status"}
+ dropped = {
+ "reason_code": "ambiguous_bare_selector",
+ "selector": "button",
+ "trajectory_index": 0,
+ }
+ scout_trajectory = [
+ {"tool_name": "click", "selector": "button", "source_url": _RESALE_URL, "trajectory_index": 0},
+ {
+ "tool_name": "click",
+ "selector": "button:nth-of-type(2)",
+ "source_url": _RESALE_URL,
+ "trajectory_index": 1,
+ },
+ ]
+
+ repair_context = workflow_update_module._ambiguous_bare_selector_repair_context(
+ code_block=code_block,
+ dropped=dropped,
+ scout_trajectory=scout_trajectory,
+ )
+
+ assert isinstance(repair_context, CodeAuthoringRepairContext)
+ assert repair_context.refiner_selector is None
+ assert repair_context.selector_alternatives == []
+
+ scout_trajectory.append(
+ {
+ "tool_name": "click",
+ "selector": 'button[data-action="status"]',
+ "source_url": _RESALE_URL,
+ "trajectory_index": 2,
+ }
+ )
+ repair_context = workflow_update_module._ambiguous_bare_selector_repair_context(
+ code_block=code_block,
+ dropped=dropped,
+ scout_trajectory=scout_trajectory,
+ )
+
+ assert isinstance(repair_context, CodeAuthoringRepairContext)
+ assert repair_context.refiner_selector == 'button[data-action="status"]'
+ assert repair_context.selector_alternatives == []
+
+ def test_ambiguous_selector_repair_context_carries_sanitized_same_page_alternatives(self) -> None:
+ code_block = {"label": "order_status"}
+ private_url = "https://example.com/orders?session=secret-token#account"
+ dropped = {
+ "reason_code": "ambiguous_bare_selector",
+ "selector": "button",
+ "trajectory_index": 1,
+ }
+ scout_trajectory = [
+ {
+ "tool_name": "type_text",
+ "selector": "#order-id",
+ "source_url": private_url,
+ "role": "textbox",
+ "trajectory_index": 0,
+ },
+ {"tool_name": "click", "selector": "button", "source_url": private_url, "trajectory_index": 1},
+ {
+ "tool_name": "click",
+ "selector": "button:nth-of-type(2)",
+ "source_url": private_url,
+ "role": "button",
+ "trajectory_index": 2,
+ },
+ {
+ "tool_name": "click",
+ "selector": 'role=button[name="Order status"]',
+ "source_url": private_url,
+ "role": "button",
+ "trajectory_index": 3,
+ },
+ {
+ "tool_name": "hover",
+ "selector": "#order-total",
+ "source_url": private_url,
+ "role": "status",
+ "trajectory_index": 4,
+ },
+ {
+ "tool_name": "click",
+ "selector": '[data-action="other-page"]',
+ "source_url": "https://example.com/account",
+ "role": "button",
+ "trajectory_index": 5,
+ },
+ ]
+
+ repair_context = workflow_update_module._ambiguous_bare_selector_repair_context(
+ code_block=code_block,
+ dropped=dropped,
+ scout_trajectory=scout_trajectory,
+ )
+
+ assert isinstance(repair_context, CodeAuthoringRepairContext)
+ assert repair_context.source_url == "https://example.com"
+ assert repair_context.refiner_selector is None
+ assert repair_context.selector_alternatives == [
+ {"tool_name": "type_text", "role": "textbox", "selector": "#order-id"},
+ {"tool_name": "click", "role": "button", "selector": 'role=button[name="Order status"]'},
+ {"tool_name": "hover", "role": "status", "selector": "#order-total"},
+ ]
+ dumped = repair_context.model_dump(mode="json")
+ assert "secret-token" not in str(dumped)
+ assert "button:nth-of-type" not in str(dumped)
+ assert "other-page" not in str(dumped)
+
+ @pytest.mark.asyncio
+ async def test_imposition_reject_replaces_stale_repair_context(self) -> None:
+ ctx = _resale_ctx(refiner_selector="button:nth-of-type(2)")
+ ctx.last_code_authoring_repair_context = _stale_unresolved_repair_context()
+
+ result = await _update_workflow({"workflow_yaml": _resale_submitted_yaml("button:nth-of-type(2)")}, ctx)
+
+ assert result["ok"] is False
+ assert result["data"]["authoring_repair_context"]["reason_code"] == "ambiguous_bare_selector"
+ assert result["data"]["authoring_repair_context"]["selector_alternatives"] == [
+ {"tool_name": "type_text", "role": "textbox", "selector": "#order-id"}
+ ]
+ assert ctx.last_code_authoring_repair_context != _stale_unresolved_repair_context()
+
class TestCodeRepairProgressClassification:
@pytest.mark.asyncio
@@ -746,6 +1036,7 @@ class TestCodeBlockParameterPersistSeam:
_stub_successful_update(monkeypatch)
ctx = _code_only_ctx()
ctx.workflow_yaml = _SAFE_CODE_YAML
+ ctx.last_code_authoring_repair_context = _stale_unresolved_repair_context()
submitted = _directory_blocks_yaml(
"""
- block_type: code
@@ -763,6 +1054,8 @@ class TestCodeBlockParameterPersistSeam:
assert "`address_or_postal_code`" in result["error"]
assert "workflow_definition.parameters" in result["error"]
assert ctx.workflow_yaml == _SAFE_CODE_YAML
+ assert "authoring_repair_context" not in result["data"]
+ assert ctx.last_code_authoring_repair_context is None
@pytest.mark.asyncio
async def test_nested_code_block_undeclared_parameter_key_rejects(self, monkeypatch: pytest.MonkeyPatch) -> None:
@@ -1104,6 +1397,7 @@ class TestCompiledAuthoringImposition:
) -> None:
_stub_successful_update(monkeypatch)
ctx = self._provider_search_ctx()
+ ctx.last_code_authoring_repair_context = _stale_unresolved_repair_context()
submitted = _yaml(
"""
title: Provider lookup
@@ -1126,6 +1420,8 @@ class TestCompiledAuthoringImposition:
assert "must pass `code_artifact_metadata`" in result["error"]
assert "goal_value_paths" in result["error"]
assert ctx.workflow_yaml == ""
+ assert "authoring_repair_context" not in result["data"]
+ assert ctx.last_code_authoring_repair_context is None
@pytest.mark.asyncio
async def test_output_intent_rejects_partial_metadata_without_output_label(
@@ -1587,12 +1883,43 @@ class TestCompiledAuthoringImposition:
) -> None:
_stub_successful_update(monkeypatch)
ctx = self._provider_search_ctx()
+ ctx.last_code_authoring_repair_context = _stale_unresolved_repair_context()
result = await _update_workflow({"workflow_yaml": _SUBMITTED_MIXED_LOCATOR_FILL_COMPUTED_LITERAL_YAML}, ctx)
assert result["ok"] is False
assert "submitted code mixes direct fills using `provider_name`" in result["error"]
assert ctx.workflow_yaml == ""
+ assert "authoring_repair_context" not in result["data"]
+ assert ctx.last_code_authoring_repair_context is None
+
+ @pytest.mark.asyncio
+ async def test_typed_default_contract_reject_clears_stale_repair_context(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = self._typed_default_ctx()
+ submitted = _yaml(
+ """
+ title: Product lookup
+ workflow_definition:
+ parameters: invalid
+ blocks:
+ - block_type: code
+ label: search_catalog
+ code: |
+ await page.locator("#search").fill("example_sku_123")
+ """
+ )
+ ctx.workflow_yaml = submitted
+ ctx.last_code_authoring_repair_context = _stale_unresolved_repair_context()
+
+ result = await _update_workflow({"workflow_yaml": submitted}, ctx)
+
+ assert result["ok"] is False
+ assert "workflow_definition.parameters must be a list" in result["error"]
+ assert "authoring_repair_context" not in result["data"]
+ assert ctx.last_code_authoring_repair_context is None
@pytest.mark.asyncio
async def test_unknown_computed_parameter_rejects_before_persist(self, monkeypatch: pytest.MonkeyPatch) -> None:
@@ -2523,12 +2850,15 @@ class TestCredentialScoutPersistGate:
async def test_rejects_credential_submit_code_without_matching_fill_scouts(self) -> None:
ctx = _code_only_ctx()
ctx.scout_trajectory = []
+ ctx.last_code_authoring_repair_context = _stale_unresolved_repair_context()
result = await _update_workflow({"workflow_yaml": self._SUBMIT_CODE_YAML}, ctx)
assert result["ok"] is False
assert "fill_credential_field" in result["error"]
assert "click the submit control or press Enter" in result["error"]
+ assert "authoring_repair_context" not in result["data"]
+ assert ctx.last_code_authoring_repair_context is None
assert result["user_facing_summary"] == (
"I need to verify the saved-credential login in the browser before I can save or run this code."
)
@@ -2813,6 +3143,10 @@ def _distinct_credential_collision_yaml(index: int) -> str:
)
+def _page_evaluate_credential_collision_yaml(index: int) -> str:
+ return _distinct_credential_collision_yaml((index * 3) + 2)
+
+
def _safe_credential_collision_yaml(index: int) -> str:
return _credential_code_yaml(
code=f"""
@@ -2909,12 +3243,51 @@ class TestCodeAuthoringGuardrailChurnBackstop:
assert ctx.latest_tool_blocker_signal is terminal
@pytest.mark.asyncio
- async def test_single_credential_priority_reject_defers_to_credential_scout_reply(self) -> None:
+ async def test_mixed_credential_and_unresolved_name_reject_returns_code_repair_progress(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ info_calls: list[tuple[str, dict[str, str | list[str]]]] = []
+
+ def capture_info(event: str, **kwargs: str | list[str]) -> None:
+ info_calls.append((event, kwargs))
+
+ monkeypatch.setattr(workflow_update_module.LOG, "info", capture_info)
ctx = _code_only_ctx()
ctx.scout_trajectory = []
result = await _update_workflow({"workflow_yaml": _distinct_credential_collision_yaml(0)}, ctx)
+ assert ctx.code_authoring_guardrail_reject_count == 1
+ assert result["ok"] is False
+ assert result["data"]["surface_kind"] == "code_repair_progress"
+ assert "failure_type" not in result["data"]
+ repair_context = result["data"]["authoring_repair_context"]
+ assert repair_context["reason_code"] == SANDBOX_UNRESOLVED_NAME_REASON_CODE
+ assert repair_context["block_label"] == "login_with_saved_credential"
+ assert repair_context["unresolved_names"] == ["undefined_helper_0"]
+ assert result["user_facing_summary"] != CREDENTIAL_SCOUT_VERIFY_REPLY
+ assert ctx.last_code_authoring_reject_was_credential_priority is False
+ assert ctx.blocker_signal is None
+ assert ctx.latest_tool_blocker_signal is None
+ assert (
+ "copilot code authoring repair context stored",
+ {
+ "reason_code": SANDBOX_UNRESOLVED_NAME_REASON_CODE,
+ "block_label": "login_with_saved_credential",
+ "unresolved_names": ["undefined_helper_0"],
+ "parameter_keys": [],
+ "available_parameter_keys": [],
+ "binding_candidates": ["undefined_helper_0"],
+ },
+ ) in info_calls
+
+ @pytest.mark.asyncio
+ async def test_single_credential_priority_reject_defers_to_credential_scout_reply(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = []
+
+ result = await _update_workflow({"workflow_yaml": _distinct_credential_collision_yaml(1)}, ctx)
+
assert ctx.code_authoring_guardrail_reject_count == 1
assert result["ok"] is False
assert result["data"]["failure_type"] == "missing_credential_or_init"
@@ -2922,13 +3295,37 @@ class TestCodeAuthoringGuardrailChurnBackstop:
assert ctx.blocker_signal is None
assert ctx.latest_tool_blocker_signal is None
+ @pytest.mark.asyncio
+ async def test_mixed_credential_and_non_name_guardrail_uses_credential_priority_path(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = []
+
+ result = await _update_workflow({"workflow_yaml": _distinct_credential_collision_yaml(1)}, ctx)
+
+ assert result["ok"] is False
+ assert result["data"]["failure_type"] == "missing_credential_or_init"
+ assert result["user_facing_summary"] == CREDENTIAL_SCOUT_VERIFY_REPLY
+ assert ctx.last_code_authoring_repair_context is None
+ assert ctx.last_code_authoring_reject_was_credential_priority is True
+
+ @pytest.mark.asyncio
+ async def test_standard_policy_mixed_credential_and_unresolved_name_omits_repair_context(self) -> None:
+ ctx = _standard_ctx()
+ ctx.scout_trajectory = []
+
+ result = await _update_workflow({"workflow_yaml": _distinct_credential_collision_yaml(0)}, ctx)
+
+ assert result["ok"] is False
+ assert "authoring_repair_context" not in result["data"]
+ assert ctx.last_code_authoring_repair_context is None
+
@pytest.mark.asyncio
async def test_credential_priority_reject_defers_below_higher_bound(self) -> None:
ctx = _code_only_ctx()
ctx.scout_trajectory = []
for index in range(MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS - 1):
- await _update_workflow({"workflow_yaml": _distinct_credential_collision_yaml(index)}, ctx)
+ await _update_workflow({"workflow_yaml": _page_evaluate_credential_collision_yaml(index)}, ctx)
assert ctx.code_authoring_guardrail_reject_count == MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS - 1
assert ctx.blocker_signal is None
@@ -2942,7 +3339,7 @@ class TestCodeAuthoringGuardrailChurnBackstop:
result: dict[str, object] = {}
for index in range(MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS):
- result = await _update_workflow({"workflow_yaml": _distinct_credential_collision_yaml(index)}, ctx)
+ result = await _update_workflow({"workflow_yaml": _page_evaluate_credential_collision_yaml(index)}, ctx)
assert ctx.code_authoring_guardrail_reject_count == MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS
assert result["ok"] is False
@@ -2960,7 +3357,7 @@ class TestCodeAuthoringGuardrailChurnBackstop:
ctx.scout_trajectory = []
for index in range(MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS):
- await _update_workflow({"workflow_yaml": _distinct_credential_collision_yaml(index)}, ctx)
+ await _update_workflow({"workflow_yaml": _page_evaluate_credential_collision_yaml(index)}, ctx)
with pytest.raises(CopilotTurnHalt) as excinfo:
_check_enforcement(ctx)
@@ -3023,7 +3420,7 @@ class TestCodeAuthoringGuardrailChurnBackstop:
ctx.scout_trajectory = []
for index in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 1):
- await _update_workflow({"workflow_yaml": _distinct_credential_collision_yaml(index)}, ctx)
+ await _update_workflow({"workflow_yaml": _page_evaluate_credential_collision_yaml(index)}, ctx)
assert ctx.last_code_authoring_reject_was_credential_priority is True
ctx.scout_trajectory = [
@@ -3184,6 +3581,22 @@ class TestBareDropSupersession:
assert workflow_update_module._selector_refines("button", 'role=button[name="x"]') is False
assert workflow_update_module._selector_refines("role=button", 'role=button[name="N"] >> nth=1') is False
+ def test_stable_bare_click_refiner_accepts_only_same_kind_text_and_role_anchors(self) -> None:
+ assert (
+ workflow_update_module._stable_same_kind_bare_click_refiner(
+ "button", "xpath=//button[normalize-space()='Check Order Status']"
+ )
+ is True
+ )
+ assert workflow_update_module._stable_same_kind_bare_click_refiner("button", 'role=button[name="Next"]') is True
+ for candidate in (
+ "xpath=//a[normalize-space()='Check Order Status']",
+ 'role=link[name="Next"]',
+ "xpath=(//button[normalize-space()='Check Order Status'])[2]",
+ "xpath=//button[contains(normalize-space(), 'Check')]",
+ ):
+ assert workflow_update_module._stable_same_kind_bare_click_refiner("button", candidate) is False
+
def test_supersession_true_returns_pairing_record(self) -> None:
dropped = {
"reason_code": "ambiguous_bare_selector",
@@ -3359,6 +3772,102 @@ class TestBareDropSupersession:
assert {record["dropped_index"] for record in forgiven} == {1, 2}
assert {record["refiner_index"] for record in forgiven} == {3, 4}
+ def test_imposition_two_exact_text_xpath_button_refiners_forgive_two_bare_drops(self) -> None:
+ ctx = _resale_ctx()
+ first_refiner = "xpath=//button[normalize-space()='Check Order Status']"
+ second_refiner = "xpath=//button[normalize-space()='View / Download']"
+ ctx.scout_trajectory = [
+ {
+ "tool_name": "type_text",
+ "selector": "#order-id",
+ "source_url": _RESALE_URL,
+ "typed_length": 6,
+ "typed_value": "abc123",
+ "trajectory_index": 0,
+ },
+ {"tool_name": "click", "selector": "button", "source_url": _RESALE_URL, "trajectory_index": 1},
+ {"tool_name": "click", "selector": "button", "source_url": _RESALE_URL, "trajectory_index": 2},
+ {"tool_name": "click", "selector": first_refiner, "source_url": _RESALE_URL, "trajectory_index": 3},
+ {"tool_name": "click", "selector": second_refiner, "source_url": _RESALE_URL, "trajectory_index": 4},
+ ]
+ submitted = _yaml(
+ """
+ title: Order status
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: order_status
+ code: |
+ await page.locator("#order-id").fill(str(order_id))
+ await page.locator("xpath=//button[normalize-space()='Check Order Status']").click()
+ await page.locator("xpath=//button[normalize-space()='View / Download']").click()
+ """
+ )
+
+ result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
+
+ assert result.violations == []
+ assert result.substitutions is not None
+ forgiven = result.substitutions["forgiven_superseded_bare_drops"]
+ assert {record["dropped_index"] for record in forgiven} == {1, 2}
+ assert {record["refiner_index"] for record in forgiven} == {3, 4}
+ assert {record["refiner_selector"] for record in forgiven} == {first_refiner, second_refiner}
+ block = _single_code_block(parse_workflow_yaml(result.workflow_yaml))
+ assert first_refiner in str(block["code"])
+ assert second_refiner in str(block["code"])
+
+ def test_imposition_one_exact_text_xpath_refiner_does_not_forgive_two_bare_drops(self) -> None:
+ ctx = _resale_ctx()
+ refiner = "xpath=//button[normalize-space()='Check Order Status']"
+ ctx.scout_trajectory = [
+ {"tool_name": "click", "selector": "#start", "source_url": _RESALE_URL, "trajectory_index": 0},
+ {"tool_name": "click", "selector": "button", "source_url": _RESALE_URL, "trajectory_index": 1},
+ {"tool_name": "click", "selector": "button", "source_url": _RESALE_URL, "trajectory_index": 2},
+ {"tool_name": "click", "selector": refiner, "source_url": _RESALE_URL, "trajectory_index": 3},
+ ]
+ submitted = _yaml(
+ """
+ title: Order status
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: order_status
+ code: |
+ await page.locator("#start").click()
+ await page.locator("xpath=//button[normalize-space()='Check Order Status']").click()
+ """
+ )
+
+ result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
+
+ assert any("ambiguous_bare_selector" in violation for violation in result.violations)
+
+ def test_imposition_positional_xpath_refiner_keeps_bare_drop_fatal(self) -> None:
+ ctx = _resale_ctx()
+ positional_refiner = "xpath=(//button[normalize-space()='Check Order Status'])[2]"
+ ctx.scout_trajectory = [
+ {"tool_name": "click", "selector": "#start", "source_url": _RESALE_URL, "trajectory_index": 0},
+ {"tool_name": "click", "selector": "button", "source_url": _RESALE_URL, "trajectory_index": 1},
+ {"tool_name": "click", "selector": positional_refiner, "source_url": _RESALE_URL, "trajectory_index": 2},
+ ]
+ submitted = _yaml(
+ """
+ title: Order status
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: order_status
+ code: |
+ await page.locator("#start").click()
+ await page.locator("xpath=(//button[normalize-space()='Check Order Status'])[2]").click()
+ """
+ )
+
+ result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
+
+ assert any("ambiguous_bare_selector" in violation for violation in result.violations)
+ assert result.substitutions is None
+
@pytest.mark.asyncio
async def test_auto_act_non_navigating_reads_role_name(self, monkeypatch: pytest.MonkeyPatch) -> None:
ctx = _auto_act_scout_ctx()
diff --git a/tests/unit/test_copilot_diagnosis_repair_contract.py b/tests/unit/test_copilot_diagnosis_repair_contract.py
index 9d65ec071..03d024319 100644
--- a/tests/unit/test_copilot_diagnosis_repair_contract.py
+++ b/tests/unit/test_copilot_diagnosis_repair_contract.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import hashlib
import json
from types import SimpleNamespace
from unittest.mock import AsyncMock
@@ -7,8 +8,9 @@ from unittest.mock import AsyncMock
import pytest
from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
+from skyvern.forge.sdk.copilot.code_block_preflight import SANDBOX_UNRESOLVED_NAME_REASON_CODE
from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict
-from skyvern.forge.sdk.copilot.context import CopilotContext
+from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext, CopilotContext
from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
DiagnosisFailureType,
RepairNextAction,
@@ -18,6 +20,7 @@ from skyvern.forge.sdk.copilot.run_outcome import (
TERMINAL_CHALLENGE_BLOCKER_REASON_CODE,
TERMINAL_CHALLENGE_RUN_OUTCOME_REASON_CODE,
)
+from skyvern.forge.sdk.copilot.tools import run_execution as run_execution_module
from skyvern.forge.sdk.copilot.turn_intent import TurnIntent, TurnIntentAuthority, TurnIntentMode
@@ -58,6 +61,17 @@ def _clean_completed_result() -> dict[str, object]:
}
+def _authoring_repair_result(repair_context: CodeAuthoringRepairContext) -> dict[str, object]:
+ return {
+ "ok": False,
+ "error": "Workflow authoring repair needed.",
+ "data": {
+ "workflow_updated": False,
+ "authoring_repair_context": repair_context.model_dump(mode="json"),
+ },
+ }
+
+
def test_contract_shapes_for_failed_suspicious_and_missing_credential_cases() -> None:
failed = build_diagnosis_repair_contract(
source_tool="run_blocks_and_collect_debug",
@@ -113,6 +127,113 @@ def test_contract_shapes_for_failed_suspicious_and_missing_credential_cases() ->
) == (DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT, RepairNextAction.ASK, ["may_answer_without_mutation"])
+def test_authoring_repair_contexts_have_distinct_structural_root_cause_signatures() -> None:
+ ambiguous = CodeAuthoringRepairContext(
+ block_label="retrieve_resale_demand_document_link",
+ reason_code="ambiguous_bare_selector",
+ selector="button",
+ refiner_selector="xpath=//button[normalize-space()='View / Download']",
+ )
+ sandbox = CodeAuthoringRepairContext(
+ block_label="retrieve_resale_demand_document_link",
+ reason_code=SANDBOX_UNRESOLVED_NAME_REASON_CODE,
+ unresolved_names=["row_text", "confirmation_number"],
+ )
+ sandbox_reordered = CodeAuthoringRepairContext(
+ block_label="retrieve_resale_demand_document_link",
+ reason_code=SANDBOX_UNRESOLVED_NAME_REASON_CODE,
+ unresolved_names=["confirmation_number", "row_text"],
+ )
+
+ ambiguous_contract = build_diagnosis_repair_contract(
+ source_tool="update_and_run_blocks",
+ result=_authoring_repair_result(ambiguous),
+ ctx=_ctx(),
+ )
+ sandbox_contract = build_diagnosis_repair_contract(
+ source_tool="update_and_run_blocks",
+ result=_authoring_repair_result(sandbox),
+ ctx=_ctx(),
+ )
+ sandbox_reordered_contract = build_diagnosis_repair_contract(
+ source_tool="update_and_run_blocks",
+ result=_authoring_repair_result(sandbox_reordered),
+ ctx=_ctx(),
+ )
+
+ ambiguous_signature = ambiguous_contract.to_trace_data()["root_cause_signature"]
+ sandbox_signature = sandbox_contract.to_trace_data()["root_cause_signature"]
+ assert ambiguous_contract.repair_decision.next_action == RepairNextAction.REPAIR
+ assert sandbox_contract.repair_decision.next_action == RepairNextAction.REPAIR
+ assert ambiguous_signature is not None
+ assert sandbox_signature is not None
+ assert ambiguous_signature != sandbox_signature
+ assert sandbox_reordered_contract.to_trace_data()["root_cause_signature"] == sandbox_signature
+
+
+def test_missing_required_output_key_repair_identity_uses_structural_context_only() -> None:
+ repair_context = CodeAuthoringRepairContext(
+ block_label="search_registry",
+ reason_code="missing_required_output_key",
+ )
+
+ contract = build_diagnosis_repair_contract(
+ source_tool="update_and_run_blocks",
+ result=_authoring_repair_result(repair_context),
+ ctx=_ctx(),
+ )
+
+ expected_payload = {
+ "version": "authoring_repair_context:v1",
+ "reason_code": "missing_required_output_key",
+ "block_label": "search_registry",
+ }
+ expected_signature = hashlib.sha256(
+ json.dumps(expected_payload, sort_keys=True, separators=(",", ":")).encode("utf-8")
+ ).hexdigest()
+
+ assert contract.repair_decision.next_action == RepairNextAction.REPAIR
+ assert contract.to_trace_data()["root_cause_signature"] == expected_signature
+ assert contract.diagnosis_result.root_cause_identity.error_class == "code_authoring_missing_required_output_key"
+
+
+def test_repair_loop_state_resets_when_authoring_repair_context_identity_changes() -> None:
+ ctx = _ctx()
+ ambiguous = CodeAuthoringRepairContext(
+ block_label="retrieve_resale_demand_document_link",
+ reason_code="ambiguous_bare_selector",
+ selector="button",
+ refiner_selector="xpath=//button[normalize-space()='View / Download']",
+ )
+ sandbox = CodeAuthoringRepairContext(
+ block_label="retrieve_resale_demand_document_link",
+ reason_code=SANDBOX_UNRESOLVED_NAME_REASON_CODE,
+ unresolved_names=["confirmation_number", "row_text"],
+ )
+
+ for expected_count in (1, 2):
+ contract = build_diagnosis_repair_contract(
+ source_tool="update_and_run_blocks",
+ result=_authoring_repair_result(ambiguous),
+ ctx=ctx,
+ )
+ run_execution_module._update_repair_loop_state(ctx, contract)
+ assert contract.repair_loop_state.consecutive_identical_repair_count == expected_count
+ assert contract.repair_loop_state.ceiling_reached is False
+
+ sandbox_contract = build_diagnosis_repair_contract(
+ source_tool="update_and_run_blocks",
+ result=_authoring_repair_result(sandbox),
+ ctx=ctx,
+ )
+ run_execution_module._update_repair_loop_state(ctx, sandbox_contract)
+
+ assert sandbox_contract.repair_decision.next_action == RepairNextAction.REPAIR
+ assert sandbox_contract.repair_loop_state.consecutive_identical_repair_count == 1
+ assert sandbox_contract.repair_loop_state.ceiling_reached is False
+ assert getattr(ctx, "blocker_signal", None) is None
+
+
def test_judge_confirmed_suspicious_success_forces_no_change() -> None:
ctx = _ctx()
ctx.last_test_suspicious_success = True
@@ -556,6 +677,149 @@ def test_unbound_credential_skip_and_parameter_binding_errors_still_ask() -> Non
)
+def test_result_unresolved_symbol_context_prefers_repair_over_credential_ask() -> None:
+ ctx = _ctx()
+ repair_context = CodeAuthoringRepairContext(
+ block_label="create_request",
+ reason_code=SANDBOX_UNRESOLVED_NAME_REASON_CODE,
+ unresolved_names=["business_name"],
+ parameter_keys=[],
+ )
+ data = {
+ "failure_type": "missing_credential_or_init",
+ "diagnostic_code_safety_errors": ["Code block references names that are unavailable."],
+ }
+ data["authoring_repair_context"] = repair_context.model_dump(mode="json")
+
+ contract = build_diagnosis_repair_contract(
+ source_tool="update_and_run_blocks",
+ result={
+ "ok": False,
+ "error": "Saved credential needs verification before running.",
+ "data": data,
+ },
+ ctx=ctx,
+ )
+
+ assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT
+ assert contract.repair_decision.next_action == RepairNextAction.REPAIR
+ assert "may_update_workflow" in contract.repair_decision.required_authority
+ assert contract.repair_decision.target_blocks == ["create_request"]
+ assert contract.to_trace_data()["next_action"] == "repair"
+
+
+def test_stale_stored_unresolved_symbol_context_does_not_override_credential_ask() -> None:
+ ctx = _ctx()
+ ctx.last_code_authoring_repair_context = CodeAuthoringRepairContext(
+ block_label="create_request",
+ reason_code=SANDBOX_UNRESOLVED_NAME_REASON_CODE,
+ unresolved_names=["business_name"],
+ parameter_keys=[],
+ )
+
+ contract = build_diagnosis_repair_contract(
+ source_tool="update_and_run_blocks",
+ result={
+ "ok": False,
+ "error": "Saved credential needs verification before running.",
+ "data": {
+ "failure_type": "missing_credential_or_init",
+ "diagnostic_code_safety_errors": ["Code block reads saved credential fields before live scouting."],
+ },
+ },
+ ctx=ctx,
+ )
+
+ assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT
+ assert contract.repair_decision.next_action == RepairNextAction.ASK
+ assert contract.repair_decision.required_authority == ["may_answer_without_mutation"]
+
+
+def test_non_credential_unresolved_name_result_repairs_instead_of_credential_ask() -> None:
+ ctx = _ctx()
+ repair_context = CodeAuthoringRepairContext(
+ block_label="create_request",
+ reason_code=SANDBOX_UNRESOLVED_NAME_REASON_CODE,
+ unresolved_names=["business_name"],
+ parameter_keys=[],
+ )
+
+ contract = build_diagnosis_repair_contract(
+ source_tool="update_and_run_blocks",
+ result={
+ "ok": False,
+ "error": "Code block `create_request` references names that are unavailable: business_name.",
+ "data": {"authoring_repair_context": repair_context.model_dump(mode="json")},
+ },
+ ctx=ctx,
+ )
+
+ assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.REPAIRABLE_BLOCK_FAILURE
+ assert contract.repair_decision.next_action == RepairNextAction.REPAIR
+ assert contract.repair_decision.target_blocks == ["create_request"]
+
+
+def test_missing_credential_without_unresolved_symbol_context_still_asks() -> None:
+ ctx = _ctx()
+ ctx.last_code_authoring_repair_context = CodeAuthoringRepairContext(
+ block_label="create_request",
+ reason_code="SANDBOX_SAFETY_CHECK",
+ unresolved_names=[],
+ parameter_keys=[],
+ )
+ contract = build_diagnosis_repair_contract(
+ source_tool="update_and_run_blocks",
+ result={
+ "ok": False,
+ "error": "Saved credential needs verification before running.",
+ "data": {"failure_type": "missing_credential_or_init"},
+ },
+ ctx=ctx,
+ )
+
+ assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT
+ assert contract.repair_decision.next_action == RepairNextAction.ASK
+ assert contract.repair_decision.required_authority == ["may_answer_without_mutation"]
+
+
+def test_unresolved_symbol_context_does_not_preempt_terminal_challenge_stop() -> None:
+ ctx = _ctx()
+ ctx.last_code_authoring_repair_context = CodeAuthoringRepairContext(
+ block_label="create_request",
+ reason_code=SANDBOX_UNRESOLVED_NAME_REASON_CODE,
+ unresolved_names=["business_name"],
+ parameter_keys=[],
+ )
+ ctx.last_test_anti_bot = "Typed run analysis reported an anti-bot challenge."
+ ctx.last_test_failure_reason = "Run output reported a blocker: Verify you are human."
+
+ contract = build_diagnosis_repair_contract(
+ source_tool="update_and_run_blocks",
+ result={
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_blocked",
+ "overall_status": "completed",
+ "failure_reason": ctx.last_test_failure_reason,
+ "failure_categories": [{"category": "ANTI_BOT_DETECTION"}],
+ "blocks": [
+ {
+ "label": "extract",
+ "block_type": "EXTRACTION",
+ "status": "completed",
+ }
+ ],
+ },
+ },
+ ctx=ctx,
+ workflow_updated=True,
+ )
+
+ assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.TERMINAL_CHALLENGE_BLOCKER
+ assert contract.repair_decision.next_action == RepairNextAction.STOP
+ assert contract.to_trace_data()["next_action"] == "stop"
+
+
def test_active_run_terminal_evidence_contract_stops_without_marking_workflow_success() -> None:
contract = build_diagnosis_repair_contract(
source_tool="update_and_run_blocks",
@@ -777,6 +1041,12 @@ def test_pre_run_challenge_observation_does_not_force_stop_on_repairable_failure
def test_post_run_gated_challenge_observation_forces_stop_on_repairable_failure() -> None:
ctx = _ctx()
+ ctx.last_code_authoring_repair_context = CodeAuthoringRepairContext(
+ block_label="create_request",
+ reason_code=SANDBOX_UNRESOLVED_NAME_REASON_CODE,
+ unresolved_names=["business_name"],
+ parameter_keys=[],
+ )
ctx.last_test_anti_bot = (
"Observed anti-bot challenge evidence before the run: challenge-gated disabled submit/search control: Search"
)
@@ -847,6 +1117,12 @@ def test_suspicious_success_flag_does_not_override_failed_run() -> None:
def test_stop_and_no_change_decisions_preserve_current_behavior_shadow_only() -> None:
stop_ctx = _ctx()
+ stop_ctx.last_code_authoring_repair_context = CodeAuthoringRepairContext(
+ block_label="create_request",
+ reason_code=SANDBOX_UNRESOLVED_NAME_REASON_CODE,
+ unresolved_names=["business_name"],
+ parameter_keys=[],
+ )
stop_ctx.last_test_non_retriable_nav_error = "net::ERR_NAME_NOT_RESOLVED"
stop_contract = build_diagnosis_repair_contract(
source_tool="run_blocks_and_collect_debug",