Skip to content

Automate browser based workflows with AI

Flag a nested helper's `global` on a block-local name and repair it as one scope defect (SKY-16213) (#8535)

Andrew Neilson34d0393e22ba
diff --git a/skyvern/cli/mcp_tools/code_block.py b/skyvern/cli/mcp_tools/code_block.py
index 9f01517fe..056c0c083 100644
--- a/skyvern/cli/mcp_tools/code_block.py
+++ b/skyvern/cli/mcp_tools/code_block.py
@@ -72,7 +72,10 @@ async def skyvern_code_block_lint(
code_safety_errors = _code_safety_errors(code)
security_errors = author_time_code_security_errors(label=label, code=code)
preflight = preflight_code_block(code, parameter_keys=keys)
- author_time = [*author_time_code_block_diagnostics(code), *await scanner_advisory_diagnostics(code)]
+ author_time = [
+ *author_time_code_block_diagnostics(code, parameter_keys=keys),
+ *await scanner_advisory_diagnostics(code),
+ ]
# Author-time diagnostics stay advisory and are deliberately excluded from ok.
ok = not (code_safety_errors or security_errors or preflight)
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index a523313c3..26236de4d 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -201,6 +201,7 @@ from skyvern.forge.sdk.copilot.runtime import (
from skyvern.forge.sdk.copilot.runtime_authoring_repair import (
OBSTRUCTION_SUMMARY_MAX_CHARS,
PAGE_VALUE_BINDING_TEXT_MAX_CHARS,
+ REPAIR_INSTRUCTION_MAX_CHARS,
RUNTIME_FAILURE_REASON_MAX_CHARS,
)
from skyvern.forge.sdk.copilot.secret_redaction import redact_raw_secrets_for_structured_prompt
@@ -1116,7 +1117,9 @@ def _code_authoring_repair_context_prompt(ctx: CopilotContext | None) -> str:
"For metadata rejects, author code_artifact_metadata with goal_value_paths, valid extraction_schema, "
"and code return paths matching required requested output child paths; rerun update_and_run_blocks."
)
- lines.append(_clean_authoring_repair_prompt_atom(repair_context.repair_instruction, max_chars=260))
+ lines.append(
+ _clean_authoring_repair_prompt_atom(repair_context.repair_instruction, max_chars=REPAIR_INSTRUCTION_MAX_CHARS)
+ )
return "\n\n" + "\n".join(line for line in lines if line)
diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.py
index 89db8c3c6..646aba0d2 100644
--- a/skyvern/forge/sdk/copilot/build_test_outcome.py
+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py
@@ -1613,6 +1613,7 @@ def recorded_outcome_from_run_blocks_result(
block_parameter_keys: Mapping[str, Sequence[str]] | None = None,
block_shape_hashes: Mapping[str, str] | None = None,
block_associations_by_label: Mapping[str, str] | None = None,
+ runtime_failure_class: str | None = None,
) -> RecordedBuildTestOutcome | None:
data = _dict(result.get("data"))
workflow_run_id = _safe_str(data.get("workflow_run_id"))
@@ -1894,7 +1895,7 @@ def recorded_outcome_from_run_blocks_result(
failure_type = _safe_str(data.get("failure_type"))
failure_categories = _failure_category_refs(carrier_backed_anti_bot_categories(data.get("failure_categories")))
status = _safe_str(failed_block.get("status")) if failed_block is not None else run_status
- runtime_failure_identity = _runtime_failure_identity(failed_block)
+ runtime_failure_identity = _runtime_failure_identity(failed_block, runtime_failure_class)
failed_operation_identity = (
_stable_hash(
{
@@ -2289,10 +2290,14 @@ def _referenced_unbound_input_keys(
return [key for key in dict.fromkeys(unbound_required_parameter_keys) if key in referenced_set]
-def _runtime_failure_identity(failed_block: Mapping[str, object] | None) -> str:
+def _runtime_failure_identity(
+ failed_block: Mapping[str, object] | None, runtime_failure_class: str | None = None
+) -> str:
if failed_block is None:
return ""
- return _locator_wait_failure_identity(failed_block) or _code_execution_failure_identity(failed_block)
+ return _locator_wait_failure_identity(failed_block) or _code_execution_failure_identity(
+ failed_block, runtime_failure_class
+ )
def _attempted_call_ref(failed_block: Mapping[str, object] | None) -> str:
@@ -2327,27 +2332,27 @@ def _runner_authored_reason(failure_reason: str) -> str:
return failure_reason.split(": ", 1)[0].strip()
-def _code_execution_failure_identity(failed_block: Mapping[str, object]) -> str:
- """Identity of a generated-code execution failure: the runner's typed error codes and the
- account of the failure the runner itself authored.
-
- Without this a raised exception carries no identity at all, so two different exceptions in the
- same block are indistinguishable and a run whose only evidence is the raise records nothing.
- """
+def _code_execution_failure_identity(
+ failed_block: Mapping[str, object], runtime_failure_class: str | None = None
+) -> str:
+ """Identity of a generated-code execution failure: the runner's typed error codes plus either the
+ classifier's ``runtime_failure_class`` or, absent one, the runner's own account of the raise."""
if not _is_code_block_failure(failed_block):
return ""
error_codes = _clean_list(_string_list(failed_block.get("error_codes")))
if not error_codes or any(code in _NOT_AUTHORED_CODE_ERROR_CODES for code in error_codes):
return ""
- return _stable_hash(
- {
- "source": "generated_code_execution",
- "error_codes": error_codes,
- "runner_reason": _runner_authored_reason(_safe_str(failed_block.get("failure_reason"))),
- "block_label": _safe_str(failed_block.get("label")),
- "block_status": _safe_str(failed_block.get("status")),
- }
- )
+ payload: dict[str, object] = {
+ "source": "generated_code_execution",
+ "error_codes": error_codes,
+ "block_label": _safe_str(failed_block.get("label")),
+ "block_status": _safe_str(failed_block.get("status")),
+ }
+ if runtime_failure_class:
+ payload["failure_class"] = _bounded_ref(runtime_failure_class)
+ else:
+ payload["runner_reason"] = _runner_authored_reason(_safe_str(failed_block.get("failure_reason")))
+ return _stable_hash(payload)
def _locator_wait_failure_identity(failed_block: Mapping[str, object]) -> str:
diff --git a/skyvern/forge/sdk/copilot/code_block_preflight.py b/skyvern/forge/sdk/copilot/code_block_preflight.py
index 087dd13aa..884626a0f 100644
--- a/skyvern/forge/sdk/copilot/code_block_preflight.py
+++ b/skyvern/forge/sdk/copilot/code_block_preflight.py
@@ -6,6 +6,7 @@ import ast
import asyncio
import keyword
import re
+import symtable
import sys
import tempfile
import textwrap
@@ -29,6 +30,7 @@ from skyvern.utils.templating import get_missing_variables
RENDER_TEMPLATE_SYNTAX_REASON_CODE = "RENDER_TEMPLATE_SYNTAX"
RENDER_UNDEFINED_NAME_REASON_CODE = "RENDER_UNDEFINED_NAME"
SCANNER_ADVISORY_REASON_CODE = "SCANNER_ADVISORY"
+WRAPPER_SCOPE_GLOBAL_REASON_CODE = "WRAPPER_SCOPE_GLOBAL"
SCANNER_ADVISORY_TIMEOUT_SECONDS = 3.0
@@ -59,6 +61,44 @@ class CodeBlockRenderDiagnostic:
failing_expression: str
+@dataclass(frozen=True)
+class NestedHelperScope:
+ """A def nested in the block body, in block line coordinates."""
+
+ name: str
+ first_line: int
+ last_line: int
+ global_names: frozenset[str]
+ nonlocal_names: frozenset[str]
+ # (name, line) of each statement that reads a name while rebinding it (`n += 1`, `n = n + 1`):
+ # the shape that means the helper wanted the enclosing value rather than a fresh local.
+ read_write_bindings: frozenset[tuple[str, int]]
+
+ def contains_line(self, line: int) -> bool:
+ return self.first_line <= line <= self.last_line
+
+
+@dataclass(frozen=True)
+class WrapperScopeGlobal:
+ """A nested ``global`` naming a name the wrapper binds: a block-body assignment or a parameter key."""
+
+ names: tuple[str, ...]
+ line: int
+ helper: NestedHelperScope
+
+
+@dataclass(frozen=True)
+class WrapperScopeFacts:
+ bound_names: frozenset[str]
+ parameter_names: frozenset[str]
+ helpers: tuple[NestedHelperScope, ...]
+ globals: tuple[WrapperScopeGlobal, ...]
+
+ def innermost_helper_at(self, line: int) -> NestedHelperScope | None:
+ candidates = [helper for helper in self.helpers if helper.contains_line(line)]
+ return min(candidates, key=lambda helper: helper.last_line - helper.first_line, default=None)
+
+
# Mirrors the runtime's strict-mode template formatter (jinja_json_finalize_strict_env)
# regardless of the WORKFLOW_TEMPLATING_STRICTNESS deployment setting.
_render_check_env = SandboxedEnvironment(undefined=StrictUndefined, finalize=_json_finalize)
@@ -231,7 +271,10 @@ _READINESS_WAIT_STATES = frozenset({"visible", "attached"})
_READINESS_EXPECTATION_METHODS = frozenset({"to_be_attached", "to_be_visible"})
# Advisory diagnostics reach the model as guidance only; keeping them out of the preflight
# gate is what stops `skyvern_code_block_lint` from turning advice into a rejection.
-_ADVISORY_DIAGNOSTIC_CODES = frozenset({"ROOT_CONTAINER_READINESS_WAIT", "ROOT_CONTAINER_TEXT_READ"})
+_ADVISORY_DIAGNOSTIC_CODES = frozenset(
+ {"ROOT_CONTAINER_READINESS_WAIT", "ROOT_CONTAINER_TEXT_READ", WRAPPER_SCOPE_GLOBAL_REASON_CODE}
+)
+_WRAPPER_FUNCTION_NAME = "__code_block__"
_WHOLE_PAGE_READ_METHODS = frozenset({"inner_text", "text_content", "all_inner_texts", "all_text_contents"})
_TABLE_ROW_TAG_SELECTOR_RE = re.compile(r"(?<![a-z0-9_-])tr(?![a-z0-9_-])")
_TABLE_ROW_ROLE_SELECTOR_RE = re.compile(r"\[role\s*=\s*(['\"]?)row\1\]")
@@ -443,17 +486,25 @@ def _may_invoke_locator_object(tree: ast.AST) -> bool:
return False
-def author_time_code_block_diagnostics(code: str) -> list[CodeBlockPreflightDiagnostic]:
+def author_time_code_block_diagnostics(
+ code: str, *, parameter_keys: Iterable[str] = ()
+) -> list[CodeBlockPreflightDiagnostic]:
tree, _ = _parse_static_ast(code)
if tree is None:
return []
- return [*_author_time_security_diagnostics(code), *_author_time_ast_diagnostics(tree)]
+ return [
+ *_author_time_security_diagnostics(code),
+ *_author_time_ast_diagnostics(tree),
+ *wrapper_scope_diagnostics(code, parameter_keys=parameter_keys),
+ ]
-def advisory_code_block_diagnostics(code: str) -> list[CodeBlockPreflightDiagnostic]:
+def advisory_code_block_diagnostics(
+ code: str, *, parameter_keys: Iterable[str] = ()
+) -> list[CodeBlockPreflightDiagnostic]:
return [
diagnostic
- for diagnostic in author_time_code_block_diagnostics(code)
+ for diagnostic in author_time_code_block_diagnostics(code, parameter_keys=parameter_keys)
if diagnostic.code in _ADVISORY_DIAGNOSTIC_CODES
]
@@ -1107,6 +1158,128 @@ def _regex_literal_diagnostic(node: ast.Call) -> CodeBlockPreflightDiagnostic |
return None
+def _read_write_bindings(own_nodes: Iterable[ast.AST], offset: int) -> frozenset[tuple[str, int]]:
+ bindings: set[tuple[str, int]] = set()
+ for node in own_nodes:
+ if isinstance(node, ast.AugAssign) and isinstance(node.target, ast.Name):
+ bindings.add((node.target.id, node.lineno + offset))
+ continue
+ if not isinstance(node, (ast.Assign, ast.AnnAssign)) or node.value is None:
+ continue
+ targets = node.targets if isinstance(node, ast.Assign) else [node.target]
+ loaded = {child.id for child in ast.walk(node.value) if isinstance(child, ast.Name)}
+ for target in targets:
+ if isinstance(target, ast.Name) and target.id in loaded:
+ bindings.add((target.id, node.lineno + offset))
+ return frozenset(bindings)
+
+
+def _own_scope_nodes(node: ast.AST) -> Iterator[ast.AST]:
+ """Descendants of ``node`` that belong to its own scope: nested defs, lambdas and classes are
+ yielded but not entered, so a statement is attributed to the innermost scope that owns it."""
+ for child in ast.iter_child_nodes(node):
+ yield child
+ if not isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda, ast.ClassDef)):
+ yield from _own_scope_nodes(child)
+
+
+def wrapper_scope_facts(code: str, *, parameter_keys: Iterable[str] = ()) -> WrapperScopeFacts | None:
+ """Scope facts about the block body as the runtime compiles it: the body of a wrapper function,
+ so every top-level binding and every parameter key is that function's local. Both executors also
+ publish parameter keys as exec globals, so a nested ``global`` on one resolves to a copy the
+ wrapper body never reads."""
+ source = _build_typed_module(code, parameter_keys=parameter_keys)
+ try:
+ tree = ast.parse(source)
+ table = symtable.symtable(source, "<code_block>", "exec")
+ except SyntaxError:
+ return None
+ wrapper_node = next(
+ (node for node in tree.body if isinstance(node, ast.AsyncFunctionDef) and node.name == _WRAPPER_FUNCTION_NAME),
+ None,
+ )
+ wrapper_table = next((child for child in table.get_children() if child.get_name() == _WRAPPER_FUNCTION_NAME), None)
+ if wrapper_node is None or wrapper_table is None:
+ return None
+ body_bound_names = frozenset(
+ symbol.get_name() for symbol in wrapper_table.get_symbols() if symbol.is_assigned() and symbol.is_local()
+ )
+ parameter_names = frozenset(
+ normalized
+ for normalized in (unicodedata.normalize("NFKC", key) for key in parameter_keys)
+ if _valid_python_identifier(normalized)
+ )
+ bound_names = body_bound_names | parameter_names
+ # The wrapper strips leading blank lines the runtime keeps, so add them back to land on the
+ # same line the runner reports.
+ dedented = textwrap.dedent(code)
+ offset = dedented[: len(dedented) - len(dedented.lstrip())].count("\n") - wrapper_node.lineno
+
+ helpers: list[NestedHelperScope] = []
+ globals_: list[WrapperScopeGlobal] = []
+ for node in ast.walk(wrapper_node):
+ if node is wrapper_node or not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+ continue
+ own_nodes = list(_own_scope_nodes(node))
+ global_statements = [child for child in own_nodes if isinstance(child, ast.Global)]
+ nonlocal_statements = [child for child in own_nodes if isinstance(child, ast.Nonlocal)]
+ helper = NestedHelperScope(
+ name=node.name,
+ first_line=node.lineno + offset,
+ last_line=(node.end_lineno or node.lineno) + offset,
+ global_names=frozenset(name for statement in global_statements for name in statement.names),
+ nonlocal_names=frozenset(name for statement in nonlocal_statements for name in statement.names),
+ read_write_bindings=_read_write_bindings(own_nodes, offset),
+ )
+ helpers.append(helper)
+ for statement in global_statements:
+ bound = tuple(name for name in statement.names if name in bound_names)
+ if bound:
+ globals_.append(WrapperScopeGlobal(names=bound, line=statement.lineno + offset, helper=helper))
+ return WrapperScopeFacts(
+ bound_names=bound_names,
+ parameter_names=parameter_names,
+ helpers=tuple(helpers),
+ globals=tuple(globals_),
+ )
+
+
+_WRAPPER_SCOPE_REMEDY = (
+ "Accumulate in a flat loop at the top level, or inside the helper declare the name `nonlocal`, "
+ "return the value, or mutate a list/dict accumulator."
+)
+
+
+def _wrapper_scope_global_message(hit: WrapperScopeGlobal, parameter_names: frozenset[str]) -> str:
+ declared = f"Code block declares `global {', '.join(hit.names)}` at line {hit.line} inside `{hit.helper.name}`"
+ if all(name in parameter_names for name in hit.names):
+ return (
+ f"{declared} for a workflow parameter. Parameters are arguments of the wrapper function the block "
+ "body runs in, so the helper's `global` reads and writes a separate module-level copy and the "
+ f"block's own reads never see the update: the run passes with stale output. {_WRAPPER_SCOPE_REMEDY}"
+ )
+ return (
+ f"{declared} for a name assigned at the block's top level. The block body runs as the body of a wrapper "
+ "function it never sees, so top-level names are that function's locals and `global` looks them "
+ "up in a module scope that never holds them: reading the name raises NameError (and "
+ "UnboundLocalError once the `global` is dropped), while a plain assignment writes that module "
+ f"scope instead, so the block silently returns the stale top-level value. {_WRAPPER_SCOPE_REMEDY}"
+ )
+
+
+def wrapper_scope_diagnostics(code: str, *, parameter_keys: Iterable[str] = ()) -> list[CodeBlockPreflightDiagnostic]:
+ facts = wrapper_scope_facts(code, parameter_keys=parameter_keys)
+ if facts is None:
+ return []
+ return [
+ CodeBlockPreflightDiagnostic(
+ code=WRAPPER_SCOPE_GLOBAL_REASON_CODE,
+ message=_wrapper_scope_global_message(hit, facts.parameter_names),
+ )
+ for hit in facts.globals
+ ]
+
+
def _build_typed_module(code: str, *, parameter_keys: Iterable[str]) -> str:
normalized_keys = dict.fromkeys(unicodedata.normalize("NFKC", key) for key in parameter_keys)
parameter_declarations = "\n".join(f"{key}: Any" for key in normalized_keys if _valid_python_identifier(key))
diff --git a/skyvern/forge/sdk/copilot/runtime_authoring_repair.py b/skyvern/forge/sdk/copilot/runtime_authoring_repair.py
index cde01383e..19d4bd98e 100644
--- a/skyvern/forge/sdk/copilot/runtime_authoring_repair.py
+++ b/skyvern/forge/sdk/copilot/runtime_authoring_repair.py
@@ -16,6 +16,7 @@ from skyvern.forge.sdk.copilot.challenge_evidence import (
is_carrier_backed_category_entry,
typed_challenge_kind,
)
+from skyvern.forge.sdk.copilot.code_block_preflight import wrapper_scope_facts
from skyvern.forge.sdk.copilot.composition_evidence import (
MAX_RESULT_CONTAINERS,
OBSERVED_CHECKED_FIELD_TYPES,
@@ -25,7 +26,7 @@ from skyvern.forge.sdk.copilot.composition_evidence import (
model_visible_composition_evidence,
)
from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, normalize_block_authoring_policy
-from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext, PageObstruction
+from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext, CopilotContext, PageObstruction
from skyvern.forge.sdk.copilot.output_contracts import code_block_available_contracts_by_label
from skyvern.forge.sdk.copilot.output_extraction_plan import candidate_relations_from_packet, page_value_binding_text
from skyvern.forge.sdk.copilot.request_policy import redact_raw_secrets_for_prompt
@@ -55,8 +56,21 @@ OBSTRUCTION_SUMMARY_MAX_CHARS = 1200
# A runner denial names its sanctioned replacement after the denied call (a listener denial runs to
# ~600 characters); a bound below that hands the repair turn the refusal without the route.
RUNTIME_FAILURE_REASON_MAX_CHARS = 640
+REPAIR_INSTRUCTION_MAX_CHARS = 260
_NO_DISMISS_CONTROL_SUMMARY = "obstruction present, no dismiss control found in page evidence"
_KEY_ERROR_RE = re.compile(r"KeyError(?:\s*:|\()\s*['\"]([^'\"]+)['\"]")
+# The runner's own reason format ("CodeBlock failed with <Class> at line N: <message>") and the
+# interpreter's quoted name in a NameError / UnboundLocalError message.
+_RUNNER_NAME_FAILURE_RE = re.compile(
+ r"CodeBlock failed with (?P<cls>NameError|UnboundLocalError) at line (?P<line>\d+): (?P<message>.*)"
+)
+_QUOTED_NAME_RE = re.compile(r"'(?P<name>[^']+)'")
+WRAPPER_SCOPE_FAILURE_CLASS = "wrapper_scope_name_resolution"
+WRAPPER_SCOPE_REPAIR_INSTRUCTION = (
+ "the block body is a wrapper function's body, so top-level names are locals `global` cannot reach; "
+ "NameError and UnboundLocalError there are one scope defect. Use a flat top-level loop, or in the "
+ "helper `nonlocal`, a return value, or a list/dict accumulator."
+)
def is_runtime_authoring_repair_context(repair_context: object) -> TypeGuard[CodeAuthoringRepairContext]:
@@ -89,14 +103,13 @@ def _missing_key_from_key_error(reason: str) -> str | None:
def _missing_output_dependency_context(
*,
- copilot_ctx: Any,
+ workflow_yaml: str | None,
block_label: str,
failed_block_status: str | None,
failure_reason: str,
run_id: str,
) -> CodeAuthoringRepairContext | None:
missing_key = _missing_key_from_key_error(failure_reason)
- workflow_yaml = getattr(copilot_ctx, "workflow_yaml", None)
if not missing_key or not isinstance(workflow_yaml, str) or not workflow_yaml.strip():
return None
contract = code_block_available_contracts_by_label(workflow_yaml).get(block_label)
@@ -134,6 +147,41 @@ def _missing_output_dependency_context(
)
+def _wrapper_scope_failure_class(workflow_yaml: str | None, block_label: str, failure_reason: str) -> str | None:
+ """NameError on a nested ``global`` for a block-bound name, or UnboundLocalError where a helper
+ declaring neither ``global`` nor ``nonlocal`` reads that name while rebinding it: one defect, the
+ block body being a wrapper function's body rather than a module. A helper that merely initializes
+ a same-named local on some paths and reads it on others has a local bug, not a scope one."""
+ match = _RUNNER_NAME_FAILURE_RE.search(failure_reason)
+ if match is None:
+ return None
+ name_match = _QUOTED_NAME_RE.search(match.group("message"))
+ if name_match is None:
+ return None
+ contract = code_block_available_contracts_by_label(workflow_yaml).get(block_label)
+ if contract is None:
+ return None
+ facts = wrapper_scope_facts(contract.code, parameter_keys=contract.parameter_keys)
+ if facts is None:
+ return None
+ name = name_match.group("name")
+ line = int(match.group("line"))
+ if name not in facts.bound_names:
+ return None
+ if match.group("cls") == "NameError":
+ if any(name in hit.names and hit.helper.contains_line(line) for hit in facts.globals):
+ return WRAPPER_SCOPE_FAILURE_CLASS
+ return None
+ helper = facts.innermost_helper_at(line)
+ if helper is None:
+ return None
+ if name in helper.global_names or name in helper.nonlocal_names:
+ return None
+ if (name, line) not in helper.read_write_bindings:
+ return None
+ return WRAPPER_SCOPE_FAILURE_CLASS
+
+
def _origin_from_runtime_url(value: Any) -> str | None:
if not isinstance(value, str) or not value.strip():
return None
@@ -547,7 +595,11 @@ def _newest_runtime_failed_block(data: dict[str, Any]) -> dict[str, Any] | None:
return None
-def record_pending_runtime_authoring_repair_context(copilot_ctx: Any, result: dict[str, Any]) -> None:
+def record_pending_runtime_authoring_repair_context(
+ copilot_ctx: CopilotContext, result: dict[str, Any], *, workflow_yaml: str | None = None
+) -> None:
+ """``workflow_yaml`` is the snapshot the run executed; the context's copy can already be a
+ later draft by the time the result lands."""
if bool(result.get("ok", False)):
clear_runtime_authoring_repair_context(copilot_ctx)
return
@@ -574,10 +626,12 @@ def record_pending_runtime_authoring_repair_context(copilot_ctx: Any, result: di
if not block_label or not failure_reason:
clear_runtime_authoring_repair_context(copilot_ctx)
return
- if is_runtime_authoring_repair_context(getattr(copilot_ctx, "last_code_authoring_repair_context", None)):
+ if is_runtime_authoring_repair_context(copilot_ctx.last_code_authoring_repair_context):
copilot_ctx.last_code_authoring_repair_context = None
+ if workflow_yaml is None:
+ workflow_yaml = copilot_ctx.workflow_yaml
missing_output_context = _missing_output_dependency_context(
- copilot_ctx=copilot_ctx,
+ workflow_yaml=workflow_yaml,
block_label=block_label,
failed_block_status=failed_block_status or None,
failure_reason=failure_reason,
@@ -586,16 +640,21 @@ def record_pending_runtime_authoring_repair_context(copilot_ctx: Any, result: di
if missing_output_context is not None:
copilot_ctx.pending_code_authoring_runtime_repair_context = missing_output_context
return
+ failure_class = _wrapper_scope_failure_class(workflow_yaml, block_label, failure_reason)
+ repair_instruction = (
+ "adapt the next code block to the observed page state and do not re-emit the same failing selector "
+ "or name path."
+ )
+ if failure_class == WRAPPER_SCOPE_FAILURE_CLASS:
+ repair_instruction = WRAPPER_SCOPE_REPAIR_INSTRUCTION
copilot_ctx.pending_code_authoring_runtime_repair_context = CodeAuthoringRepairContext(
block_label=block_label,
reason_code=_RUNTIME_AUTHORING_REASON_CODE,
runtime_failure_reason=failure_reason,
+ runtime_failure_class=failure_class,
failed_block_status=failed_block_status or None,
workflow_run_id=run_id,
- repair_instruction=(
- "adapt the next code block to the observed page state and do not re-emit the same failing selector "
- "or name path."
- ),
+ repair_instruction=repair_instruction,
)
diff --git a/skyvern/forge/sdk/copilot/tools/banned_blocks.py b/skyvern/forge/sdk/copilot/tools/banned_blocks.py
index 7a0dbd601..829a7078e 100644
--- a/skyvern/forge/sdk/copilot/tools/banned_blocks.py
+++ b/skyvern/forge/sdk/copilot/tools/banned_blocks.py
@@ -295,9 +295,8 @@ def _code_only_browser_unavailable_summary() -> str:
def _code_only_browser_validation_guidance() -> str:
return (
- "Use validate_block only for allowed non-browser helper blocks. Do not use validate_block for `code` "
- "blocks, dummy/probe code blocks, or browser/page native block types; validate real code blocks through "
- "update_and_run_blocks."
+ "validate_block is only for allowed non-browser helper blocks, never for `code` blocks, dummy/probe "
+ "code blocks, or browser/page native block types; validate real code blocks through update_and_run_blocks."
)
@@ -337,6 +336,12 @@ def _code_only_browser_schema_guidance() -> list[str]:
]
+WRAPPER_SCOPE_RUNTIME_FACT = """\
+- The body runs inside a wrapper function: identifier parameter keys and top-level names are
+ its locals, so `global` never reaches them. Accumulate in a flat loop; a nested helper
+ updates one with `nonlocal`, a return value, or a mutable accumulator."""
+
+
def _code_only_browser_authoring_prompt() -> str:
pending = "\n".join(f"- {detail}" for detail in _code_only_browser_pending_details())
return f"""
@@ -345,13 +350,10 @@ ACTIVE BLOCK AUTHORING POLICY: CODE-ONLY BROWSER MODE
{_code_only_browser_unavailable_summary()}
Rules:
-- Browser/page/session durable steps must be focused `code` blocks.
-- In code-only browser mode, before authoring the first `code` block this turn,
- call `get_block_schema` with `block_type: code` and follow its returned field
- names and nesting exactly; do not guess the YAML shape from memory.
-- Allowed non-browser helper blocks remain available: `conditional`, `for_loop`,
- `while_loop`, `send_email`, `human_interaction`, S3/Google Sheets helpers, file
- parsers, and triggers.
+- Before authoring the first `code` block this turn, call `get_block_schema` with `block_type: code`
+ and follow its returned field names and nesting exactly; do not guess the YAML shape.
+- Non-browser helper blocks stay available: `conditional`, `for_loop`, `while_loop`,
+ `send_email`, `human_interaction`, S3/Google Sheets helpers, file parsers, and triggers.
- {_code_only_browser_validation_guidance()}
Code-native capabilities still pending plumbing:
@@ -359,17 +361,18 @@ Code-native capabilities still pending plumbing:
Runtime facts:
- `code` is async Python with a Playwright `page` object and workflow parameters by key.
-- The runtime pre-injects its helper namespaces; do not write `import` statements and do
- not access dunder (`__name__`) names or attributes.
-- Valid Python identifier parameter keys are local variables; normalize values before page inputs.
+- Helper namespaces are pre-injected: no `import` statements, no dunder (`__name__`) names
+ or attributes.
+{WRAPPER_SCOPE_RUNTIME_FACT}
+- Normalize parameter values before page inputs.
- Use deterministic, bounded Playwright calls and selectors observed while scouting.
- For browser reads, prefer visible anchors, locator text, block outputs, and
MCP/scout evidence gathered before authoring.
- Return JSON-safe structured data plus visible evidence text for records, totals,
confirmations, and identifiers.
-- For an extraction-intent `code` block, derive a typed `extraction_schema` (named
- fields with types) from the goal and the scouted page, carry it as
- `code_artifact_metadata.extraction_schema`, and conform the block's `return` to it.
+- For an extraction-intent `code` block, derive a typed `extraction_schema` from the goal
+ and the scouted page, carry it as `code_artifact_metadata.extraction_schema`, and
+ conform the block's `return` to it.
- Use YAML block scalars (`code: |`) and pass complete workflow YAML to update tools.
""".strip()
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index 11d8bde82..dac17d6dc 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -162,6 +162,7 @@ from skyvern.forge.sdk.copilot.runtime import (
from skyvern.forge.sdk.copilot.runtime_authoring_repair import (
build_test_page_state_from_evidence,
inject_runtime_authoring_repair_context,
+ is_runtime_authoring_repair_context,
post_run_inspection_cleanly_matches,
record_pending_runtime_authoring_repair_context,
repair_page_evidence_is_admissible,
@@ -4767,7 +4768,11 @@ def _record_run_blocks_result(
copilot_ctx.post_run_page_observation_workflow_run_id = None
copilot_ctx.post_run_page_observation_after_failed_test = False
copilot_ctx.post_run_current_page_inspection_workflow_run_id = None
- record_pending_runtime_authoring_repair_context(copilot_ctx, result)
+ record_pending_runtime_authoring_repair_context(
+ copilot_ctx,
+ result,
+ workflow_yaml=result.execution.workflow_yaml if isinstance(result, _ExecutionResult) else None,
+ )
structured_blocker = _run_blocks_structured_blocker_message(result, copilot_ctx)
anti_bot_match, empty_data_blocks, failure_categories, goal_path_omissions = _analyze_run_blocks(
@@ -5018,6 +5023,21 @@ def _record_executed_block_labels(copilot_ctx: CopilotContext, result: dict[str,
copilot_ctx.executed_block_fingerprints.setdefault(label, set()).update(block_fingerprints)
+def _same_run_runtime_failure_class(copilot_ctx: CopilotContext, workflow_run_id: str | None) -> str | None:
+ """The classifier's verdict on this run's failure, whether it is still pending or already
+ finalized with page evidence, so an in-turn regrade keys the same identity as the first record;
+ a later read whose turn contexts are already cleared has no verdict left to key on."""
+ if not workflow_run_id:
+ return None
+ for repair_context in (
+ copilot_ctx.pending_code_authoring_runtime_repair_context,
+ copilot_ctx.last_code_authoring_repair_context,
+ ):
+ if is_runtime_authoring_repair_context(repair_context) and repair_context.workflow_run_id == workflow_run_id:
+ return repair_context.runtime_failure_class
+ return None
+
+
def _build_recorded_build_test_outcome(
copilot_ctx: CopilotContext,
result: dict[str, Any],
@@ -5068,6 +5088,9 @@ def _build_recorded_build_test_outcome(
return recorded_outcome_from_run_blocks_result(
result,
page_evidence=result_page_evidence or copilot_ctx.composition_page_evidence,
+ runtime_failure_class=_same_run_runtime_failure_class(
+ copilot_ctx, _packet_string(result_data.get("workflow_run_id"))
+ ),
recorded_run_outcome=recorded_run_outcome,
completion_verification=None,
authored_structure_signature=authored_structure_signature_from_workflow(
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index 3dff44c81..9e246134e 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -40,6 +40,8 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (
)
from skyvern.forge.sdk.copilot.canonical_ownership import workflow_content_fingerprint
from skyvern.forge.sdk.copilot.code_block_preflight import (
+ WRAPPER_SCOPE_GLOBAL_REASON_CODE,
+ CodeBlockPreflightDiagnostic,
advisory_code_block_diagnostics,
scanner_advisory_diagnostics,
)
@@ -66,6 +68,7 @@ from skyvern.forge.sdk.copilot.google_connection_notice import (
)
from skyvern.forge.sdk.copilot.narration import CODE_REPAIR_PROGRESS_SURFACE_KIND, CODE_REPAIR_PROGRESS_TEXT
from skyvern.forge.sdk.copilot.output_contracts import (
+ code_block_available_contracts_by_label,
declared_string_workflow_parameter_keys,
declared_workflow_parameter_keys,
)
@@ -758,16 +761,21 @@ def _changed_code_blocks(prior_yaml: str | None, submitted_yaml: str, accepted_y
return changed
-def _advisory_labels_by_message(changed_code_blocks: Mapping[str, str]) -> dict[str, list[str]]:
- """Labels per advisory message, computed from every changed block rather than the
+def _advisory_labels_by_diagnostic(
+ changed_code_blocks: Mapping[str, str], accepted_yaml: str
+) -> dict[CodeBlockPreflightDiagnostic, list[str]]:
+ """Labels per advisory diagnostic, computed from every changed block rather than the
budget-truncated ``stored_code`` so an oversized block still gets its note."""
- labels_by_message: dict[str, list[str]] = {}
+ contracts = code_block_available_contracts_by_label(accepted_yaml)
+ labels_by_diagnostic: dict[CodeBlockPreflightDiagnostic, list[str]] = {}
for label, code in sorted(changed_code_blocks.items()):
- for diagnostic in advisory_code_block_diagnostics(code):
- labels = labels_by_message.setdefault(diagnostic.message, [])
+ contract = contracts.get(label)
+ parameter_keys = contract.parameter_keys if contract is not None else ()
+ for diagnostic in advisory_code_block_diagnostics(code, parameter_keys=parameter_keys):
+ labels = labels_by_diagnostic.setdefault(diagnostic, [])
if label not in labels:
labels.append(label)
- return labels_by_message
+ return labels_by_diagnostic
async def _scanner_advisory_labels_by_message(
@@ -3786,6 +3794,12 @@ _PERSISTENCE_MESSAGES: dict[PersistenceDisposition, str] = {
}
+READINESS_WAIT_ADVISORY_REASON_CODE = "code_block_readiness_wait_advisory"
+# A nested `global` fails every run, but the run reports NameError at the use site and cannot
+# report that the block body is compiled inside a wrapper function the author never sees.
+WRAPPER_SCOPE_ADVISORY_REASON_CODE = "code_block_wrapper_scope_advisory"
+
+
def persistence_disposition(ctx: AgentContext) -> PersistenceDisposition:
return "staged_auto_apply" if ctx.auto_accept is True else "staged"
@@ -3816,7 +3830,7 @@ def _author_time_findings(
*,
schema_incompatibility: SchemaIncompatibility | None,
metadata_violations: Sequence[str],
- code_block_diagnostics: Mapping[str, list[str]] | None = None,
+ code_block_diagnostics: Mapping[CodeBlockPreflightDiagnostic, list[str]] | None = None,
scanner_diagnostics: Mapping[str, list[str]] | None = None,
) -> list[dict[str, Any]]:
"""Non-blocking labels on a draft that persisted anyway. Each entry needs a reason a
@@ -3841,18 +3855,20 @@ def _author_time_findings(
"summary": "\n".join(str(violation) for violation in metadata_violations),
}
)
- # A contentless readiness wait is intermittent by construction: it passes on every run where the
- # page happens to settle, so a green test-run cannot tell the author the wait encodes nothing.
- if code_block_diagnostics:
- findings.append(
- {
- "reason_code": "code_block_readiness_wait_advisory",
- "summary": "\n".join(
- f"Code blocks {', '.join(f'`{label}`' for label in labels)}: {message}"
- for message, labels in code_block_diagnostics.items()
- ),
- }
+ # Neither defect is visible to a green test-run: a contentless readiness wait passes whenever the
+ # page happens to settle, and a write-only wrapper-scope `global` silently drops the update.
+ summaries_by_reason_code: dict[str, list[str]] = {}
+ for diagnostic, labels in (code_block_diagnostics or {}).items():
+ reason_code = (
+ WRAPPER_SCOPE_ADVISORY_REASON_CODE
+ if diagnostic.code == WRAPPER_SCOPE_GLOBAL_REASON_CODE
+ else READINESS_WAIT_ADVISORY_REASON_CODE
+ )
+ summaries_by_reason_code.setdefault(reason_code, []).append(
+ f"Code blocks {', '.join(f'`{label}`' for label in labels)}: {diagnostic.message}"
)
+ for reason_code, summaries in summaries_by_reason_code.items():
+ findings.append({"reason_code": reason_code, "summary": "\n".join(summaries)})
# A scanner-flagged pattern is invisible to a test-run by construction: the code runs and
# succeeds — that is exactly what makes the flagged behavior worth a warning to the author.
if scanner_diagnostics:
@@ -4480,7 +4496,7 @@ async def _update_workflow(
# Best-effort — the workflow is already persisted by this point, so an advisory that trips on
# crafted block code must never turn a successful update into a failed turn.
try:
- advisory_labels = _advisory_labels_by_message(changed_code_blocks)
+ advisory_labels = _advisory_labels_by_diagnostic(changed_code_blocks, workflow_yaml)
except Exception as advisory_err:
LOG.warning("copilot_advisory_code_block_diagnostics_failed", error=str(advisory_err))
advisory_labels = {}
diff --git a/tests/unit/fixtures/copilot/wrapper_scope_exception/packet-name-error.json b/tests/unit/fixtures/copilot/wrapper_scope_exception/packet-name-error.json
new file mode 100644
index 000000000..0b7a6d8e3
--- /dev/null
+++ b/tests/unit/fixtures/copilot/wrapper_scope_exception/packet-name-error.json
@@ -0,0 +1,38 @@
+{
+ "schema": "generated-code-exception-packet-v1",
+ "provenance": "runner reason format and failing-line geometry from a recorded failure; the block is reduced to its scoping shape with generic names, selectors and text, and the reason is regenerated by executing that shape under the runtime wrapper geometry with a stub page",
+ "runner_exception_class": "builtins.NameError",
+ "runner_failing_line": 48,
+ "expected_runtime_failure_class": "wrapper_scope_name_resolution",
+ "workflow_yaml": "title: Catalog section items\nworkflow_definition:\n parameters:\n - parameter_type: workflow\n key: setup_only\n workflow_parameter_type: boolean\n blocks:\n - block_type: code\n label: open_and_play_catalog_items\n parameter_keys:\n - setup_only\n code: \"async def open_next_at_section_start(section_name):\\n next_button = page.locator(\\\"a.nav-forward\\\")\\n if not await next_button.count() or not await next_button.first.is_visible():\\n raise RuntimeError(f\\\"The Next button was not visible at the start of {section_name}\\\")\\n await next_button.first.click()\\n await page.wait_for_timeout(1000)\\n\\nawait open_next_at_section_start(\\\"Section Two\\\")\\n\\nsection_item_links = page.locator(\\\"#panel-alpha > aside > div:nth-of-type(5) a[href*='/items/']\\\")\\nlink_count = await section_item_links.count()\\nif link_count == 0:\\n raise RuntimeError(\\\"No items were visible in the observed Section Two\\\")\\n\\nitem_urls = []\\nfor index in range(link_count):\\n href = await section_item_links.nth(index).get_attribute(\\\"href\\\")\\n if href and \\\"/items/\\\" in href and href not in item_urls:\\n item_urls.append(href)\\n\\nitems_completed = 0\\nitems_started = 0\\npages_visited = 0\\nitem_titles = []\\n\\nasync def play_item_if_present():\\n global items_completed, items_started\\n play_button = page.locator(\\\"button[aria-label='Play'][title='Play']\\\")\\n progress = page.locator(\\\"[role='slider'][aria-label='Progress']\\\")\\n await page.wait_for_timeout(1200)\\n if not await play_button.count() or not await progress.count():\\n return False\\n if not await play_button.first.is_visible() or not await progress.first.is_visible():\\n return False\\n settings_button = page.locator(\\\"button[aria-label='Open options']\\\")\\n if await settings_button.count() and await settings_button.first.is_visible():\\n await settings_button.first.click()\\n await page.wait_for_timeout(250)\\n rate_menu = page.locator(\\\"#menu-beta\\\")\\n if await rate_menu.count() and await rate_menu.first.is_visible():\\n await rate_menu.first.click()\\n await page.wait_for_timeout(250)\\n fast_label = page.locator(\\\"label[for^='opt-gamma-']\\\")\\n if await fast_label.count() and await fast_label.first.is_visible():\\n await fast_label.first.click()\\n await page.wait_for_timeout(200)\\n await play_button.first.click()\\n items_started += 1\\n title = (await page.locator(\\\"body\\\").inner_text())[:160]\\n item_titles.append(title)\\n if bool(setup_only):\\n return True\\n duration_value = await progress.first.get_attribute(\\\"aria-valuemax\\\")\\n if not duration_value:\\n raise RuntimeError(\\\"The observed progress control did not expose an item duration\\\")\\n duration_seconds = float(duration_value)\\n completed = False\\n for _ in range(int(duration_seconds / 60) + 10):\\n await page.wait_for_timeout(30000)\\n position_value = await progress.first.get_attribute(\\\"aria-valuenow\\\")\\n if position_value and float(position_value) >= duration_seconds - 2:\\n completed = True\\n break\\n if not completed:\\n raise RuntimeError(\\\"An item did not reach completion during bounded playback checks\\\")\\n items_completed += 1\\n return True\\n\\nfor item_url in item_urls:\\n await page.goto(\\\"https://example.com\\\" + item_url, wait_until=\\\"domcontentloaded\\\")\\n pages_visited += 1\\n played = await play_item_if_present()\\n if played and bool(setup_only):\\n return {\\\"setup_confirmed\\\": True, \\\"intro_skipped\\\": True, \\\"current_item_url\\\": page.url, \\\"items_started\\\": items_started, \\\"items_completed\\\": items_completed, \\\"pages_visited\\\": pages_visited, \\\"visible_evidence\\\": (await page.locator(\\\"body\\\").inner_text())[:800]}\\n\\nextras_section_url = \\\"https://example.com/catalog/sections/extras#view\\\"\\nawait page.goto(extras_section_url, wait_until=\\\"domcontentloaded\\\")\\nawait page.wait_for_timeout(800)\\nextras_links = page.locator(\\\"#panel-alpha > aside > div:nth-of-type(7) a[href*='/items/']\\\")\\nextras_count = await extras_links.count()\\nextras_urls = []\\nfor index in range(extras_count):\\n href = await extras_links.nth(index).get_attribute(\\\"href\\\")\\n if href and \\\"/items/\\\" in href and href not in extras_urls:\\n extras_urls.append(href)\\n\\nextras_pages_visited = 0\\nfor item_url in extras_urls:\\n await page.goto(\\\"https://example.com\\\" + item_url, wait_until=\\\"domcontentloaded\\\")\\n extras_pages_visited += 1\\n played = await play_item_if_present()\\n if played and bool(setup_only):\\n return {\\\"setup_confirmed\\\": True, \\\"intro_skipped\\\": True, \\\"current_item_url\\\": page.url, \\\"items_started\\\": items_started, \\\"items_completed\\\": items_completed, \\\"pages_visited\\\": pages_visited, \\\"extras_pages_visited\\\": extras_pages_visited, \\\"visible_evidence\\\": (await page.locator(\\\"body\\\").inner_text())[:800]}\\n\\nreturn {\\\"setup_confirmed\\\": items_started > 0, \\\"intro_skipped\\\": True, \\\"items_started\\\": items_started, \\\"items_completed\\\": items_completed, \\\"pages_visited\\\": pages_visited, \\\"extras_pages_visited\\\": extras_pages_visited, \\\"final_item_url\\\": page.url, \\\"visible_evidence\\\": (await page.locator(\\\"body\\\").inner_text())[:800]}\\n\"\n",
+ "result": {
+ "ok": false,
+ "data": {
+ "workflow_run_id": "wr_wrapper_scope_fixture",
+ "browser_session_id": "pbs_wrapper_scope_fixture",
+ "overall_status": "failed",
+ "requested_block_labels": [
+ "open_and_play_catalog_items"
+ ],
+ "executed_block_labels": [
+ "open_and_play_catalog_items"
+ ],
+ "post_run_page_capture": {
+ "status": "unavailable",
+ "omission": "page_capture_unavailable"
+ },
+ "blocks": [
+ {
+ "label": "open_and_play_catalog_items",
+ "block_type": "CODE",
+ "status": "failed",
+ "workflow_run_block_id": "wrb_wrapper_scope_fixture",
+ "failure_reason": "CodeBlock failed with NameError at line 48: name 'items_started' is not defined.",
+ "error_codes": [
+ "user_code_error"
+ ]
+ }
+ ]
+ }
+ }
+}
diff --git a/tests/unit/fixtures/copilot/wrapper_scope_exception/packet-unbound-local-error.json b/tests/unit/fixtures/copilot/wrapper_scope_exception/packet-unbound-local-error.json
new file mode 100644
index 000000000..4d75b5181
--- /dev/null
+++ b/tests/unit/fixtures/copilot/wrapper_scope_exception/packet-unbound-local-error.json
@@ -0,0 +1,38 @@
+{
+ "schema": "generated-code-exception-packet-v1",
+ "provenance": "reconstruction: the same block with its `global` line dropped, executed under the runtime wrapper geometry with a stub page on CPython 3.13; not custody of a recorded repair attempt",
+ "runner_exception_class": "builtins.UnboundLocalError",
+ "runner_failing_line": 47,
+ "expected_runtime_failure_class": "wrapper_scope_name_resolution",
+ "workflow_yaml": "title: Catalog section items\nworkflow_definition:\n parameters:\n - parameter_type: workflow\n key: setup_only\n workflow_parameter_type: boolean\n blocks:\n - block_type: code\n label: open_and_play_catalog_items\n parameter_keys:\n - setup_only\n code: \"async def open_next_at_section_start(section_name):\\n next_button = page.locator(\\\"a.nav-forward\\\")\\n if not await next_button.count() or not await next_button.first.is_visible():\\n raise RuntimeError(f\\\"The Next button was not visible at the start of {section_name}\\\")\\n await next_button.first.click()\\n await page.wait_for_timeout(1000)\\n\\nawait open_next_at_section_start(\\\"Section Two\\\")\\n\\nsection_item_links = page.locator(\\\"#panel-alpha > aside > div:nth-of-type(5) a[href*='/items/']\\\")\\nlink_count = await section_item_links.count()\\nif link_count == 0:\\n raise RuntimeError(\\\"No items were visible in the observed Section Two\\\")\\n\\nitem_urls = []\\nfor index in range(link_count):\\n href = await section_item_links.nth(index).get_attribute(\\\"href\\\")\\n if href and \\\"/items/\\\" in href and href not in item_urls:\\n item_urls.append(href)\\n\\nitems_completed = 0\\nitems_started = 0\\npages_visited = 0\\nitem_titles = []\\n\\nasync def play_item_if_present():\\n play_button = page.locator(\\\"button[aria-label='Play'][title='Play']\\\")\\n progress = page.locator(\\\"[role='slider'][aria-label='Progress']\\\")\\n await page.wait_for_timeout(1200)\\n if not await play_button.count() or not await progress.count():\\n return False\\n if not await play_button.first.is_visible() or not await progress.first.is_visible():\\n return False\\n settings_button = page.locator(\\\"button[aria-label='Open options']\\\")\\n if await settings_button.count() and await settings_button.first.is_visible():\\n await settings_button.first.click()\\n await page.wait_for_timeout(250)\\n rate_menu = page.locator(\\\"#menu-beta\\\")\\n if await rate_menu.count() and await rate_menu.first.is_visible():\\n await rate_menu.first.click()\\n await page.wait_for_timeout(250)\\n fast_label = page.locator(\\\"label[for^='opt-gamma-']\\\")\\n if await fast_label.count() and await fast_label.first.is_visible():\\n await fast_label.first.click()\\n await page.wait_for_timeout(200)\\n await play_button.first.click()\\n items_started += 1\\n title = (await page.locator(\\\"body\\\").inner_text())[:160]\\n item_titles.append(title)\\n if bool(setup_only):\\n return True\\n duration_value = await progress.first.get_attribute(\\\"aria-valuemax\\\")\\n if not duration_value:\\n raise RuntimeError(\\\"The observed progress control did not expose an item duration\\\")\\n duration_seconds = float(duration_value)\\n completed = False\\n for _ in range(int(duration_seconds / 60) + 10):\\n await page.wait_for_timeout(30000)\\n position_value = await progress.first.get_attribute(\\\"aria-valuenow\\\")\\n if position_value and float(position_value) >= duration_seconds - 2:\\n completed = True\\n break\\n if not completed:\\n raise RuntimeError(\\\"An item did not reach completion during bounded playback checks\\\")\\n items_completed += 1\\n return True\\n\\nfor item_url in item_urls:\\n await page.goto(\\\"https://example.com\\\" + item_url, wait_until=\\\"domcontentloaded\\\")\\n pages_visited += 1\\n played = await play_item_if_present()\\n if played and bool(setup_only):\\n return {\\\"setup_confirmed\\\": True, \\\"intro_skipped\\\": True, \\\"current_item_url\\\": page.url, \\\"items_started\\\": items_started, \\\"items_completed\\\": items_completed, \\\"pages_visited\\\": pages_visited, \\\"visible_evidence\\\": (await page.locator(\\\"body\\\").inner_text())[:800]}\\n\\nextras_section_url = \\\"https://example.com/catalog/sections/extras#view\\\"\\nawait page.goto(extras_section_url, wait_until=\\\"domcontentloaded\\\")\\nawait page.wait_for_timeout(800)\\nextras_links = page.locator(\\\"#panel-alpha > aside > div:nth-of-type(7) a[href*='/items/']\\\")\\nextras_count = await extras_links.count()\\nextras_urls = []\\nfor index in range(extras_count):\\n href = await extras_links.nth(index).get_attribute(\\\"href\\\")\\n if href and \\\"/items/\\\" in href and href not in extras_urls:\\n extras_urls.append(href)\\n\\nextras_pages_visited = 0\\nfor item_url in extras_urls:\\n await page.goto(\\\"https://example.com\\\" + item_url, wait_until=\\\"domcontentloaded\\\")\\n extras_pages_visited += 1\\n played = await play_item_if_present()\\n if played and bool(setup_only):\\n return {\\\"setup_confirmed\\\": True, \\\"intro_skipped\\\": True, \\\"current_item_url\\\": page.url, \\\"items_started\\\": items_started, \\\"items_completed\\\": items_completed, \\\"pages_visited\\\": pages_visited, \\\"extras_pages_visited\\\": extras_pages_visited, \\\"visible_evidence\\\": (await page.locator(\\\"body\\\").inner_text())[:800]}\\n\\nreturn {\\\"setup_confirmed\\\": items_started > 0, \\\"intro_skipped\\\": True, \\\"items_started\\\": items_started, \\\"items_completed\\\": items_completed, \\\"pages_visited\\\": pages_visited, \\\"extras_pages_visited\\\": extras_pages_visited, \\\"final_item_url\\\": page.url, \\\"visible_evidence\\\": (await page.locator(\\\"body\\\").inner_text())[:800]}\\n\"\n",
+ "result": {
+ "ok": false,
+ "data": {
+ "workflow_run_id": "wr_wrapper_scope_fixture",
+ "browser_session_id": "pbs_wrapper_scope_fixture",
+ "overall_status": "failed",
+ "requested_block_labels": [
+ "open_and_play_catalog_items"
+ ],
+ "executed_block_labels": [
+ "open_and_play_catalog_items"
+ ],
+ "post_run_page_capture": {
+ "status": "unavailable",
+ "omission": "page_capture_unavailable"
+ },
+ "blocks": [
+ {
+ "label": "open_and_play_catalog_items",
+ "block_type": "CODE",
+ "status": "failed",
+ "workflow_run_block_id": "wrb_wrapper_scope_fixture",
+ "failure_reason": "CodeBlock failed with UnboundLocalError at line 47: cannot access local variable 'items_started' where it is not associated with a value.",
+ "error_codes": [
+ "user_code_error"
+ ]
+ }
+ ]
+ }
+ }
+}
diff --git a/tests/unit/test_code_block_lint_mcp_tool.py b/tests/unit/test_code_block_lint_mcp_tool.py
index 5a32daf70..b21f03a8e 100644
--- a/tests/unit/test_code_block_lint_mcp_tool.py
+++ b/tests/unit/test_code_block_lint_mcp_tool.py
@@ -116,6 +116,16 @@ async def test_body_readiness_advisory_warns_without_failing_the_lint_gate() ->
assert _has_diagnostic(result, section="author_time_diagnostics", code="ROOT_CONTAINER_READINESS_WAIT")
+async def test_wrapper_scope_advisory_sees_the_lint_calls_parameter_keys() -> None:
+ code = "async def bump():\n global retries\n retries += 1\n\nawait bump()\nreturn {}"
+
+ result = await skyvern_code_block_lint(code=code, parameter_keys=["retries"])
+
+ assert result["data"]["lint_ok"] is True
+ assert _has_diagnostic(result, section="author_time_diagnostics", code="WRAPPER_SCOPE_GLOBAL")
+
+
def _install_scanner_stub(
monkeypatch: pytest.MonkeyPatch,
findings: list[CodeBlockScanFinding] | Exception,
diff --git a/tests/unit/test_code_block_preflight.py b/tests/unit/test_code_block_preflight.py
index 5617b4d99..e5b8f99e5 100644
--- a/tests/unit/test_code_block_preflight.py
+++ b/tests/unit/test_code_block_preflight.py
@@ -6,19 +6,109 @@ OSS-synced: only example.* placeholder targets and synthetic labels.
from __future__ import annotations
import ast
+import textwrap
import pytest
from skyvern.forge.sdk.copilot.code_block_preflight import (
RENDER_TEMPLATE_SYNTAX_REASON_CODE,
RENDER_UNDEFINED_NAME_REASON_CODE,
+ WRAPPER_SCOPE_GLOBAL_REASON_CODE,
CodeBlockRenderDiagnostic,
_build_typed_module,
+ advisory_code_block_diagnostics,
code_block_render_diagnostic,
+ wrapper_scope_diagnostics,
+ wrapper_scope_facts,
)
_BOUND_NAMES = frozenset({"business_name", "contact_email", "submit_request", "submit_request_output"})
+# The shape of a real saved block: top-level counters, a nested helper that declares them
+# ``global`` and increments them, and a loop calling the helper. Selectors and text are generic.
+_GLOBAL_IN_HELPER_HEAD = """\
+links = page.locator("nav a[href*='/items/']")
+link_count = await links.count()
+if link_count == 0:
+ raise RuntimeError("No items were visible")
+
+item_urls = []
+for index in range(link_count):
+ href = await links.nth(index).get_attribute("href")
+ if href and href not in item_urls:
+ item_urls.append(href)
+
+items_completed = 0
+items_started = 0
+titles = []
+
+async def open_item_if_present():
+"""
+_GLOBAL_IN_HELPER_TAIL = """\
+ play_button = page.locator("button[aria-label='Play']")
+ if not await play_button.count():
+ return False
+ await play_button.first.click()
+ items_started += 1
+ titles.append(await page.title())
+ if bool(setup_only):
+ return True
+ await page.wait_for_timeout(500)
+ items_completed += 1
+ return True
+
+for item_url in item_urls:
+ await page.goto("https://example.com" + item_url, wait_until="domcontentloaded")
+ opened = await open_item_if_present()
+ if opened and bool(setup_only):
+ return {"items_started": items_started, "items_completed": items_completed}
+
+return {"items_started": items_started, "items_completed": items_completed, "titles": titles}
+"""
+_GLOBAL_IN_HELPER = _GLOBAL_IN_HELPER_HEAD + " global items_completed, items_started\n" + _GLOBAL_IN_HELPER_TAIL
+_NONLOCAL_IN_HELPER = _GLOBAL_IN_HELPER_HEAD + " nonlocal items_completed, items_started\n" + _GLOBAL_IN_HELPER_TAIL
+_RETURN_VALUE_HELPER = """\
+items_started = 0
+
+async def open_item():
+ await page.locator("button[aria-label='Play']").first.click()
+ return 1
+
+for _ in range(3):
+ items_started += await open_item()
+return {"items_started": items_started}
+"""
+_ACCUMULATOR_HELPER = """\
+counts = {"started": 0}
+
+async def open_item(acc):
+ await page.locator("button[aria-label='Play']").first.click()
+ acc["started"] += 1
+
+for _ in range(3):
+ await open_item(counts)
+return counts
+"""
+_GLOBAL_FOR_UNBOUND_NAME = """\
+async def helper():
+ global request_count
+ request_count = 1
+
+await helper()
+return {"ok": True}
+"""
+_WRAPPER_LEVEL_GLOBAL = """\
+global items_started
+items_started = 0
+
+async def helper():
+ global items_started
+ items_started += 1
+
+await helper()
+return {"items_started": items_started}
+"""
+
class TestCodeBlockRenderDiagnosticRejects:
def test_parameters_namespace_reference_is_unrenderable(self) -> None:
@@ -114,3 +204,82 @@ def test_preflight_declares_normalized_parameters_once_and_omits_normalized_keyw
assert declared.count("file") == 1
assert "if" not in declared
assert "__private" not in declared
+
+
+class TestWrapperScopeGlobalAdvisory:
+ def test_global_for_a_top_level_name_in_a_nested_helper_is_flagged_with_its_line(self) -> None:
+ diagnostics = advisory_code_block_diagnostics(_GLOBAL_IN_HELPER)
+ scope = [diagnostic for diagnostic in diagnostics if diagnostic.code == WRAPPER_SCOPE_GLOBAL_REASON_CODE]
+ assert len(scope) == 1
+ assert "`global items_completed, items_started` at line 17 inside `open_item_if_present`" in scope[0].message
+
+ def test_global_line_follows_the_runtime_when_the_block_starts_with_blank_lines(self) -> None:
+ diagnostics = wrapper_scope_diagnostics("\n\n" + _GLOBAL_IN_HELPER)
+ assert len(diagnostics) == 1
+ assert "at line 19 inside" in diagnostics[0].message
+
+ @pytest.mark.parametrize(
+ "code",
+ [
+ _NONLOCAL_IN_HELPER,
+ _RETURN_VALUE_HELPER,
+ _ACCUMULATOR_HELPER,
+ _GLOBAL_FOR_UNBOUND_NAME,
+ _WRAPPER_LEVEL_GLOBAL,
+ ],
+ ids=["nonlocal", "return_value", "mutable_accumulator", "never_bound_name", "wrapper_level_global"],
+ )
+ def test_sanctioned_and_exact_negative_shapes_yield_no_advisory(self, code: str) -> None:
+ assert [d for d in advisory_code_block_diagnostics(code) if d.code == WRAPPER_SCOPE_GLOBAL_REASON_CODE] == []
+
+ def test_a_global_for_a_parameter_key_is_advised_only_when_the_keys_are_known(self) -> None:
+ code = "async def helper():\n global setup_only\n setup_only = False\n\nawait helper()\nreturn {}\n"
+ assert advisory_code_block_diagnostics(code) == []
+ diagnostics = advisory_code_block_diagnostics(code, parameter_keys=["setup_only"])
+ assert [d.code for d in diagnostics] == [WRAPPER_SCOPE_GLOBAL_REASON_CODE]
+ assert "`global setup_only` at line 2 inside `helper` for a workflow parameter" in diagnostics[0].message
+ assert "NameError" not in diagnostics[0].message
+
+ def test_a_mixed_global_statement_names_only_the_block_bound_name(self) -> None:
+ code = textwrap.dedent(
+ """\
+ count = 0
+
+ async def helper():
+ global count, phantom
+ count += 1
+
+ await helper()
+ return {"count": count}
+ """
+ )
+ diagnostics = advisory_code_block_diagnostics(code)
+ assert [d.code for d in diagnostics] == [WRAPPER_SCOPE_GLOBAL_REASON_CODE]
+ assert "`global count` at line 4 inside `helper`" in diagnostics[0].message
+ assert "phantom" not in diagnostics[0].message
+
+ def test_a_helper_nested_in_a_helper_yields_one_diagnostic_owned_by_the_inner_helper(self) -> None:
+ code = textwrap.dedent(
+ """\
+ count = 0
+
+ async def outer():
+ async def inner():
+ global count
+ count += 1
+
+ await inner()
+
+ await outer()
+ return {"count": count}
+ """
+ )
+ diagnostics = advisory_code_block_diagnostics(code)
+ assert [d.code for d in diagnostics] == [WRAPPER_SCOPE_GLOBAL_REASON_CODE]
+ assert "`global count` at line 5 inside `inner`" in diagnostics[0].message
+ facts = wrapper_scope_facts(code)
+ assert facts is not None
+ assert {helper.name: helper.global_names for helper in facts.helpers} == {
+ "outer": frozenset(),
+ "inner": frozenset({"count"}),
+ }
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index 4158a051a..9adac1c25 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -106,6 +106,7 @@ from skyvern.forge.sdk.copilot.request_policy import (
from skyvern.forge.sdk.copilot.request_slots import PROMPT_NAME as REQUEST_SLOTS_PROMPT_NAME
from skyvern.forge.sdk.copilot.review_gate import workflow_block_fingerprints
from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome, interim_run_start_outcome
+from skyvern.forge.sdk.copilot.runtime_authoring_repair import REPAIR_INSTRUCTION_MAX_CHARS
from skyvern.forge.sdk.copilot.tools import _run_blocks_and_collect_debug
from skyvern.forge.sdk.copilot.tools import credentials as credentials_module
from skyvern.forge.sdk.copilot.tools import run_execution as run_execution_module
@@ -915,6 +916,22 @@ workflow_definition:
assert replacement in prompt
+ def test_the_repair_instruction_renders_to_the_shared_instruction_budget(self) -> None:
+ instruction = "adapt the next code block: " + "w" * (2 * REPAIR_INSTRUCTION_MAX_CHARS)
+ ctx = _ctx(
+ block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER,
+ last_code_authoring_repair_context=CodeAuthoringRepairContext(
+ block_label="collect_rows",
+ reason_code="runtime_block_failure",
+ repair_instruction=instruction,
+ ),
+ )
+
+ prompt = agent_module._code_authoring_repair_context_prompt(ctx)
+
+ assert instruction[:REPAIR_INSTRUCTION_MAX_CHARS] in prompt
+ assert instruction[: REPAIR_INSTRUCTION_MAX_CHARS + 1] not in prompt
+
def test_metadata_repair_context_prompt_includes_failure_and_contract_guidance(self) -> None:
long_reason = "missing requested output child paths " + ("x" * 700)
repair_context = CodeAuthoringRepairContext(
@@ -4507,7 +4524,7 @@ class TestCredentialRefusalReachesAgent:
assert "ACTIVE BLOCK AUTHORING POLICY: CODE-ONLY BROWSER MODE" in prompt
assert "credential-typed code" in prompt
assert "download registration" in prompt
- assert "Use validate_block only for allowed non-browser helper blocks" in prompt
+ assert "validate_block is only for allowed non-browser helper blocks" in prompt
assert "Do not call `validate_block`" not in prompt
assert "native_allowed" not in prompt
diff --git a/tests/unit/test_copilot_banned_block_policy.py b/tests/unit/test_copilot_banned_block_policy.py
index 6ff43ff49..0247958c5 100644
--- a/tests/unit/test_copilot_banned_block_policy.py
+++ b/tests/unit/test_copilot_banned_block_policy.py
@@ -536,7 +536,7 @@ async def test_code_schema_guidance_is_policy_rendered_and_allows_helper_validat
out = await _get_block_schema_post_hook(result, raw={}, ctx=code_only_ctx)
assert "Browser/page workflow block types are unavailable" in out["data"]["code_only_note"]
- assert "validate_block only for allowed non-browser helper blocks" in " ".join(out["data"]["code_only_guidance"])
+ assert "validate_block is only for allowed non-browser helper blocks" in " ".join(out["data"]["code_only_guidance"])
assert "Do not persist navigation/action/login" not in " ".join(out["data"]["code_only_guidance"])
diff --git a/tests/unit/test_copilot_build_test_outcome.py b/tests/unit/test_copilot_build_test_outcome.py
index db186eb13..5c94103c9 100644
--- a/tests/unit/test_copilot_build_test_outcome.py
+++ b/tests/unit/test_copilot_build_test_outcome.py
@@ -2216,6 +2216,44 @@ def test_unavailable_registered_output_rows_do_not_become_false_missing_output_f
assert any("persisted registered output values were unavailable" in notice for notice in packet.omission_notices)
+def _code_exception_result(failure_reason: str) -> dict[str, object]:
+ return {
+ "ok": False,
+ "data": {
+ "workflow_run_id": "wr_code_exception",
+ "overall_status": "failed",
+ "blocks": [
+ {
+ "label": "count_items",
+ "block_type": "CODE",
+ "status": "failed",
+ "failure_reason": failure_reason,
+ "error_codes": ["user_code_error"],
+ }
+ ],
+ },
+ }
+
+
+def test_a_classified_code_exception_keys_on_its_class_instead_of_the_runner_reason() -> None:
+ name_error = _code_exception_result("CodeBlock failed with NameError at line 48: name 'count' is not defined.")
+ unbound = _code_exception_result(
+ "CodeBlock failed with UnboundLocalError at line 47: cannot access local variable 'count' where it is not "
+ "associated with a value."
+ )
+
+ unclassified = [recorded_outcome_from_run_blocks_result(result) for result in (name_error, unbound)]
+ classified = [
+ recorded_outcome_from_run_blocks_result(result, runtime_failure_class="wrapper_scope_name_resolution")
+ for result in (name_error, unbound)
+ ]
+
+ assert all(outcome is not None and outcome.structural_key for outcome in [*unclassified, *classified])
+ assert unclassified[0].structural_key != unclassified[1].structural_key
+ assert classified[0].structural_key == classified[1].structural_key
+ assert classified[0].structural_key != unclassified[0].structural_key
+
+
def test_runtime_block_failure_outcome_keys_playwright_hidden_locator_structure() -> None:
table_result = {
"ok": False,
diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.py
index 5052e2973..bbc98d254 100644
--- a/tests/unit/test_copilot_code_block_persist_seam.py
+++ b/tests/unit/test_copilot_code_block_persist_seam.py
@@ -27,12 +27,15 @@ from skyvern.forge.sdk.copilot.request_policy import (
from skyvern.forge.sdk.copilot.secret_scrub import REDACTED_SECRET_PLACEHOLDER, register_secret_scrub_value
from skyvern.forge.sdk.copilot.tools import workflow_update as workflow_update_module
from skyvern.forge.sdk.copilot.tools.workflow_update import (
+ READINESS_WAIT_ADVISORY_REASON_CODE,
+ WRAPPER_SCOPE_ADVISORY_REASON_CODE,
CodeArtifactCompletionCriterion,
_accepted_code_delta,
- _advisory_labels_by_message,
+ _advisory_labels_by_diagnostic,
_author_time_findings,
_changed_code_blocks,
_update_workflow,
+ carry_author_time_findings,
)
from skyvern.forge.sdk.copilot.workflow_credential_utils import parse_workflow_yaml, workflow_blocks
from skyvern.forge.sdk.copilot.workflow_yaml import delete_block_from_workflow
@@ -733,7 +736,9 @@ class TestBodyReadinessAdvisoryDelivery:
return _author_time_findings(
schema_incompatibility=None,
metadata_violations=[],
- code_block_diagnostics=_advisory_labels_by_message(_changed_code_blocks(prior, accepted, accepted)),
+ code_block_diagnostics=_advisory_labels_by_diagnostic(
+ _changed_code_blocks(prior, accepted, accepted), accepted
+ ),
)
def test_budget_withheld_block_still_carries_the_advisory(self) -> None:
@@ -753,6 +758,94 @@ class TestBodyReadinessAdvisoryDelivery:
assert self._findings(prior, prior) == []
+async def test_wrapper_scope_advisory_is_its_own_finding_on_a_draft_that_persists_and_runs(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ persisted: list[str] = []
+ _stub_successful_update(monkeypatch, persisted)
+ ctx = _ctx()
+ submitted = _code_yaml(
+ """
+ await page.locator("body").wait_for(state="visible", timeout=45000)
+ matches = 0
+
+ async def count_match():
+ global matches
+ matches += 1
+
+ for _ in range(3):
+ await count_match()
+ return {"output": {"matches": matches}}
+ """
+ )
+
+ result = await _update_workflow(
+ {"workflow_yaml": submitted, "code_artifact_metadata": []},
+ ctx,
+ allow_missing_credentials=True,
+ )
+
+ assert result["ok"] is True
+ assert persisted == [submitted]
+ findings = result["data"]["findings"]
+ assert [finding["reason_code"] for finding in findings] == [
+ READINESS_WAIT_ADVISORY_REASON_CODE,
+ WRAPPER_SCOPE_ADVISORY_REASON_CODE,
+ ]
+ assert "`global matches` at line 5 inside `count_match`" in findings[1]["summary"]
+ assert "`submit_search`" in findings[1]["summary"]
+ run_result = {"ok": False, "data": {"workflow_run_id": "wr_x", "overall_status": "failed"}}
+ carried = carry_author_time_findings(result, run_result)["data"]
+ assert carried["findings"] == findings
+ assert carried["workflow_run_id"] == "wr_x"
+
+
+async def test_wrapper_scope_advisory_covers_a_global_on_a_declared_workflow_parameter(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ persisted: list[str] = []
+ _stub_successful_update(monkeypatch, persisted)
+ ctx = _ctx()
+ code = "\n".join(
+ [
+ " async def bump():",
+ " global retries",
+ " retries += 1",
+ "",
+ " await bump()",
+ ' return {"output": {"retries": retries}}',
+ ]
+ )
+ submitted = (
+ "title: Search\n"
+ "workflow_definition:\n"
+ " parameters:\n"
+ " - parameter_type: workflow\n"
+ " workflow_parameter_type: integer\n"
+ " key: retries\n"
+ " default_value: 0\n"
+ " blocks:\n"
+ " - block_type: code\n"
+ " label: submit_search\n"
+ " parameter_keys: [retries]\n"
+ " code: |\n"
+ f"{code}\n"
+ )
+
+ result = await _update_workflow(
+ {"workflow_yaml": submitted, "code_artifact_metadata": []},
+ ctx,
+ allow_missing_credentials=True,
+ )
+
+ assert result["ok"] is True
+ findings = result["data"]["findings"]
+ assert [finding["reason_code"] for finding in findings] == [WRAPPER_SCOPE_ADVISORY_REASON_CODE]
+ assert "`global retries` at line 2 inside `bump` for a workflow parameter" in findings[0]["summary"]
+
+
@pytest.mark.asyncio
async def test_scanner_advisory_findings_never_block_a_save(monkeypatch: pytest.MonkeyPatch) -> None:
persisted: list[str] = []
diff --git a/tests/unit/test_copilot_enforcement_pruning.py b/tests/unit/test_copilot_enforcement_pruning.py
index caf56d3f4..6061f9128 100644
--- a/tests/unit/test_copilot_enforcement_pruning.py
+++ b/tests/unit/test_copilot_enforcement_pruning.py
@@ -415,6 +415,8 @@ def _fresh_ctx_for_record() -> SimpleNamespace:
last_run_blocks_block_labels=[],
last_run_outcome=None,
narrative_block_attempts={},
+ last_code_authoring_repair_context=None,
+ pending_code_authoring_runtime_repair_context=None,
)
diff --git a/tests/unit/test_copilot_generated_browser_page_state_repair.py b/tests/unit/test_copilot_generated_browser_page_state_repair.py
index b9dc21968..ecc54a4ad 100644
--- a/tests/unit/test_copilot_generated_browser_page_state_repair.py
+++ b/tests/unit/test_copilot_generated_browser_page_state_repair.py
@@ -1,9 +1,16 @@
from __future__ import annotations
+import asyncio
+import copy
import json
+import textwrap
+from collections.abc import Generator
from pathlib import Path
+from types import SimpleNamespace
+from typing import Any
import pytest
+import yaml
from skyvern.forge.sdk.copilot.agent import (
_build_user_context,
@@ -17,21 +24,32 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (
)
from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext, CopilotContext
+from skyvern.forge.sdk.copilot.output_contracts import code_block_available_contracts_by_label
from skyvern.forge.sdk.copilot.output_utils import (
_compact_packet_for_aggregate_limit,
project_build_test_packet_for_llm,
project_direct_test_handoff_packet_for_llm,
)
from skyvern.forge.sdk.copilot.runtime_authoring_repair import (
+ REPAIR_INSTRUCTION_MAX_CHARS,
+ WRAPPER_SCOPE_FAILURE_CLASS,
+ WRAPPER_SCOPE_REPAIR_INSTRUCTION,
finalize_runtime_authoring_repair_context_from_page_observation,
record_pending_runtime_authoring_repair_context,
repair_page_evidence_is_admissible,
)
from skyvern.forge.sdk.copilot.tools.run_execution import (
+ CopilotExecutionSnapshot,
+ _build_recorded_build_test_outcome,
+ _ExecutionResult,
_failure_action_trace_summary,
_newest_failed_result,
+ _record_run_blocks_result,
+ _RunExecution,
build_test_evidence_packet,
)
+from skyvern.forge.sdk.workflow.models.code_block_recorder import CODE_BLOCK_FILENAME, user_code_line_from_exception
+from skyvern.forge.sdk.workflow.models.workflow import Workflow
_RUN_ID = "wr_analytics_scalar"
_RUN_BROWSER_SESSION_ID = "pbs_run_visible"
@@ -43,6 +61,9 @@ _OVERLAY_ID = "notice-overlay"
_RESTOCK_NOTICE_TEXT = "We will let you know when this option is available again."
_COMPLETED_PATH = "visitors"
_CONSENT_PACKET_PATH = Path(__file__).resolve().parent / "fixtures/copilot/consent_cover_repair/packet.json"
+_WRAPPER_SCOPE_PACKET_DIR = Path(__file__).resolve().parent / "fixtures/copilot/wrapper_scope_exception"
+_WRAPPER_SCOPE_LABEL = "open_and_play_catalog_items"
+_WRAPPER_SCOPE_GLOBAL_LINE = "global items_completed, items_started"
_CONSENT_BLOCK_LABEL = "extract_order_documents"
_CONSENT_LAYER_TEXT = "Terms of Service"
_CANDIDATE_RUN_ID = "wr_candidate_price_scope"
@@ -536,3 +557,288 @@ def test_runtime_repair_context_names_the_failure_the_run_stopped_on() -> None:
pending = ctx.pending_code_authoring_runtime_repair_context
assert pending is not None
assert pending.block_label == "select_first_result"
+
+
+class _StubPage:
+ """Every attribute is another stub, every call returns one, and awaiting one yields one, so
+ the block runs to its scope failure without a browser."""
+
+ url = "https://example.com/items/1"
+
+ def __getattr__(self, name: str) -> _StubPage:
+ return _StubPage()
+
+ def __call__(self, *args: Any, **kwargs: Any) -> _StubPage:
+ return _StubPage()
+
+ def __await__(self) -> Generator[None, None, _StubPage]:
+ yield from ()
+ return _StubPage()
+
+ def __index__(self) -> int:
+ return 1
+
+ def __contains__(self, item: object) -> bool:
+ return True
+
+ def __add__(self, other: object) -> str:
+ return self.url
+
+ def __radd__(self, other: object) -> str:
+ return self.url
+
+
+def _runner_reason_from_executing(code: str) -> str:
+ """The runner's reason for this block, from actually running it as the runtime does: the
+ block body as the body of a wrapper function, under the runner's filename and line offset."""
+ full_code = "\nasync def wrapper():\n" + textwrap.indent(textwrap.dedent(code), " ") + "\n"
+ namespace: dict[str, Any] = {"page": _StubPage(), "setup_only": True}
+ exec(compile(full_code, CODE_BLOCK_FILENAME, "exec"), namespace, namespace) # noqa: S102
+ try:
+ asyncio.run(namespace["wrapper"]())
+ except Exception as exc:
+ return f"CodeBlock failed with {type(exc).__name__} at line {user_code_line_from_exception(exc)}: {exc}."
+ raise AssertionError("the wrapped block ran without raising")
+
+
+def _wrapper_scope_packet(name: str) -> dict[str, Any]:
+ return json.loads((_WRAPPER_SCOPE_PACKET_DIR / name).read_text(encoding="utf-8"))
+
+
+def _wrapper_scope_result(failure_reason: str) -> dict[str, Any]:
+ return {
+ "ok": False,
+ "data": {
+ "workflow_run_id": "wr_wrapper_scope_fixture",
+ "overall_status": "failed",
+ "blocks": [{"label": _WRAPPER_SCOPE_LABEL, "status": "failed", "failure_reason": failure_reason}],
+ },
+ }
+
+
+def _pending_after(result: dict[str, Any], workflow_yaml: str) -> CodeAuthoringRepairContext:
+ ctx = _copilot_context()
+ record_pending_runtime_authoring_repair_context(ctx, result, workflow_yaml=workflow_yaml)
+ pending = ctx.pending_code_authoring_runtime_repair_context
+ assert pending is not None
+ return pending
+
+
+def _with_block_code(workflow_yaml: str, code: str) -> str:
+ document = yaml.safe_load(workflow_yaml)
+ block = next(b for b in document["workflow_definition"]["blocks"] if b["label"] == _WRAPPER_SCOPE_LABEL)
+ assert block["code"] != code
+ block["code"] = code
+ edited = yaml.safe_dump(document, sort_keys=False)
+ assert code_block_available_contracts_by_label(edited)[_WRAPPER_SCOPE_LABEL].code == code
+ return edited
+
+
+def _as_second_run(result: dict[str, Any]) -> dict[str, Any]:
+ second = copy.deepcopy(result)
+ second["data"]["workflow_run_id"] = "wr_wrapper_scope_fixture_second"
+ return second
+
+
+def _record_run(ctx: CopilotContext, result: dict[str, Any], workflow_yaml: str) -> tuple[str, object]:
+ ctx.workflow_yaml = workflow_yaml
+ _record_run_blocks_result(ctx, result)
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None and outcome.reason_code == "runtime_block_failure"
+ return outcome.structural_failure_identity, ctx.recorded_build_test_outcome_history[-1]["structural_key"]
+
+
+def test_name_error_then_unbound_local_error_on_the_same_counter_are_one_scope_defect() -> None:
+ name_error = _wrapper_scope_packet("packet-name-error.json")
+ unbound = _wrapper_scope_packet("packet-unbound-local-error.json")
+ code_with_global = code_block_available_contracts_by_label(name_error["workflow_yaml"])[_WRAPPER_SCOPE_LABEL].code
+ code_without_global = code_block_available_contracts_by_label(unbound["workflow_yaml"])[_WRAPPER_SCOPE_LABEL].code
+ assert _WRAPPER_SCOPE_GLOBAL_LINE in code_with_global and _WRAPPER_SCOPE_GLOBAL_LINE not in code_without_global
+ assert (
+ _runner_reason_from_executing(code_with_global) == name_error["result"]["data"]["blocks"][0]["failure_reason"]
+ )
+ executed_reason = _runner_reason_from_executing(code_without_global)
+ assert executed_reason.startswith("CodeBlock failed with UnboundLocalError at line 47: ")
+ assert executed_reason == unbound["result"]["data"]["blocks"][0]["failure_reason"]
+
+ first = _pending_after(name_error["result"], name_error["workflow_yaml"])
+ second = _pending_after(_wrapper_scope_result(executed_reason), unbound["workflow_yaml"])
+
+ assert first.runtime_failure_class == WRAPPER_SCOPE_FAILURE_CLASS
+ assert second.runtime_failure_class == WRAPPER_SCOPE_FAILURE_CLASS
+ assert first.repair_instruction == second.repair_instruction == WRAPPER_SCOPE_REPAIR_INSTRUCTION
+ assert len(WRAPPER_SCOPE_REPAIR_INSTRUCTION) <= REPAIR_INSTRUCTION_MAX_CHARS
+
+
+def test_the_second_scope_exception_records_the_same_outcome_identity_as_the_first() -> None:
+ name_error = _wrapper_scope_packet("packet-name-error.json")
+ unbound = _wrapper_scope_packet("packet-unbound-local-error.json")
+ ctx = _copilot_context()
+
+ first_identity, first_key = _record_run(ctx, name_error["result"], name_error["workflow_yaml"])
+ second_identity, second_key = _record_run(ctx, _as_second_run(unbound["result"]), unbound["workflow_yaml"])
+
+ assert first_identity and first_identity == second_identity
+ assert first_key is not None and first_key == second_key
+ assert len(ctx.recorded_build_test_outcome_history) == 2
+
+
+def test_a_regrade_after_the_repair_context_is_finalized_keeps_the_first_identity() -> None:
+ name_error = _wrapper_scope_packet("packet-name-error.json")
+ ctx = _copilot_context()
+ first_identity, _ = _record_run(ctx, name_error["result"], name_error["workflow_yaml"])
+ ctx.last_code_authoring_repair_context = ctx.pending_code_authoring_runtime_repair_context
+ ctx.pending_code_authoring_runtime_repair_context = None
+
+ regraded = _build_recorded_build_test_outcome(ctx, name_error["result"], None)
+
+ assert regraded is not None and regraded.structural_failure_identity == first_identity
+
+
+def test_an_unrelated_name_error_keeps_its_own_outcome_identity() -> None:
+ name_error = _wrapper_scope_packet("packet-name-error.json")
+ unbound = _wrapper_scope_packet("packet-unbound-local-error.json")
+ ctx = _copilot_context()
+ typo = copy.deepcopy(name_error["result"])
+ typo["data"]["blocks"][0]["failure_reason"] = (
+ "CodeBlock failed with NameError at line 48: name 'typo' is not defined."
+ )
+
+ typo_identity, typo_key = _record_run(ctx, typo, name_error["workflow_yaml"])
+ scope_identity, scope_key = _record_run(ctx, _as_second_run(unbound["result"]), unbound["workflow_yaml"])
+
+ assert typo_identity != scope_identity
+ assert typo_key != scope_key
+
+
+def test_an_inner_helpers_global_does_not_hide_the_outer_helpers_scope_defect() -> None:
+ packet = _wrapper_scope_packet("packet-name-error.json")
+ code = textwrap.dedent(
+ """\
+ count = 0
+
+ async def outer():
+ async def inner():
+ global count
+ count = 5
+
+ await inner()
+ count += 1
+
+ await outer()
+ return {"count": count}
+ """
+ )
+ reason = _runner_reason_from_executing(code)
+ assert reason.startswith("CodeBlock failed with UnboundLocalError at line 9: ")
+
+ pending = _pending_after(_wrapper_scope_result(reason), _with_block_code(packet["workflow_yaml"], code))
+
+ assert pending.runtime_failure_class == WRAPPER_SCOPE_FAILURE_CLASS
+
+
+def test_executed_snapshot_yaml_outranks_the_context_draft_for_scope_classification() -> None:
+ packet = _wrapper_scope_packet("packet-name-error.json")
+ fixed_yaml = _with_block_code(packet["workflow_yaml"], 'items_started = 0\nreturn {"items_started": 0}\n')
+ ctx = _copilot_context()
+ ctx.workflow_yaml = fixed_yaml
+
+ record_pending_runtime_authoring_repair_context(ctx, packet["result"])
+ from_draft = ctx.pending_code_authoring_runtime_repair_context
+ record_pending_runtime_authoring_repair_context(ctx, packet["result"], workflow_yaml=packet["workflow_yaml"])
+ from_snapshot = ctx.pending_code_authoring_runtime_repair_context
+
+ assert from_draft is not None and from_draft.runtime_failure_class is None
+ assert from_snapshot is not None and from_snapshot.runtime_failure_class == WRAPPER_SCOPE_FAILURE_CLASS
+
+
+def _execution_result(result: dict[str, Any], executed_workflow_yaml: str) -> _ExecutionResult:
+ workflow = Workflow.model_construct(workflow_id="wf_fixture", workflow_definition=SimpleNamespace(blocks=[]))
+ execution = _RunExecution(
+ snapshot=CopilotExecutionSnapshot(
+ provenance="staged",
+ workflow=workflow,
+ workflow_parameters=(),
+ output_parameters=(),
+ workflow_yaml=executed_workflow_yaml,
+ ),
+ workflow_yaml=executed_workflow_yaml,
+ metadata={},
+ associations={},
+ source_at_start=None,
+ unbound_keys=[],
+ explicit_blank=False,
+ )
+ return _ExecutionResult(result, execution)
+
+
+def test_recording_an_execution_result_classifies_against_the_snapshot_it_ran_not_the_context_draft() -> None:
+ packet = _wrapper_scope_packet("packet-name-error.json")
+ ctx = _copilot_context()
+ ctx.workflow_yaml = _with_block_code(packet["workflow_yaml"], 'items_started = 0\nreturn {"items_started": 0}\n')
+
+ _record_run_blocks_result(ctx, _execution_result(packet["result"], packet["workflow_yaml"]))
+
+ pending = ctx.pending_code_authoring_runtime_repair_context
+ assert pending is not None and pending.runtime_failure_class == WRAPPER_SCOPE_FAILURE_CLASS
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None and outcome.reason_code == "runtime_block_failure"
+
+
+ ("failure_reason", "code_edit"),
+ [
+ ("CodeBlock failed with NameError at line 48: name 'typo' is not defined.", None),
+ (
+ "CodeBlock failed with NameError at line 48: name 'typo_total' is not defined.",
+ (" global items_completed, items_started\n", " global items_completed, items_started, typo_total\n"),
+ ),
+ ("CodeBlock failed with NameError at line 12: name 'items_started' is not defined.", None),
+ (
+ "CodeBlock failed with NameError at line 48: name 'never_bound' is not defined.",
+ (" global items_completed, items_started\n", " global never_bound\n"),
+ ),
+ (
+ "CodeBlock failed with UnboundLocalError at line 47: cannot access local variable 'items_started' "
+ "where it is not associated with a value.",
+ (" global items_completed, items_started\n", " nonlocal items_completed, items_started\n"),
+ ),
+ (
+ "CodeBlock failed with UnboundLocalError at line 47: cannot access local variable 'never_bound' "
+ "where it is not associated with a value.",
+ None,
+ ),
+ (
+ "CodeBlock failed with UnboundLocalError at line 29: cannot access local variable 'pages_visited' "
+ "where it is not associated with a value.",
+ (
+ " global items_completed, items_started\n",
+ " if link_count:\n pages_visited = 1\n print(pages_visited)\n",
+ ),
+ ),
+ ],
+ ids=[
+ "unrelated_name_with_dormant_global",
+ "never_bound_name_sharing_a_global_with_a_bound_one",
+ "name_error_outside_the_helper",
+ "never_bound_name",
+ "nonlocal_present",
+ "unbound_local_on_a_never_bound_name",
+ "conditionally_initialized_local_sharing_a_top_level_name",
+ ],
+)
+def test_scope_class_needs_the_failing_name_and_line_to_land_in_the_helper(
+ failure_reason: str, code_edit: tuple[str, str] | None
+) -> None:
+ packet = _wrapper_scope_packet("packet-name-error.json")
+ workflow_yaml = packet["workflow_yaml"]
+ if code_edit is not None:
+ code = code_block_available_contracts_by_label(workflow_yaml)[_WRAPPER_SCOPE_LABEL].code
+ assert code_edit[0] in code
+ workflow_yaml = _with_block_code(workflow_yaml, code.replace(code_edit[0], code_edit[1]))
+
+ pending = _pending_after(_wrapper_scope_result(failure_reason), workflow_yaml)
+
+ assert pending.reason_code == "runtime_block_failure"
+ assert pending.runtime_failure_class is None
+ assert "wrapper" not in pending.repair_instruction
diff --git a/tests/unit/test_workflow_copilot_prompt_rewrite.py b/tests/unit/test_workflow_copilot_prompt_rewrite.py
index e5862b9ae..640ace1c3 100644
--- a/tests/unit/test_workflow_copilot_prompt_rewrite.py
+++ b/tests/unit/test_workflow_copilot_prompt_rewrite.py
@@ -148,7 +148,7 @@ async def test_workflow_knowledge_marks_code_only_policy_as_authoritative() -> N
def test_code_only_policy_is_short_and_contains_no_settled_block_conversion_steering() -> None:
rendered = _code_only_browser_authoring_prompt()
- assert "before authoring the first `code` block" in rendered
+ assert "Before authoring the first `code` block" in rendered
assert "call `get_block_schema` with `block_type: code`" in rendered
assert "solve_captcha" not in rendered
assert "<key>.username" not in rendered