Skip to content

Automate browser based workflows with AI

SKY-12933 Copilot: build the login step from what the scout demonstrated (#7713)

Andrew Neilsonbcf74309237a
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 05610e39f..c893b8f0e 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -102,7 +102,10 @@ from skyvern.forge.sdk.copilot.context import (
render_loaded_result_context_for_prompt,
sanitize_global_llm_context_for_prompt,
)
-from skyvern.forge.sdk.copilot.credential_literal_rebind import rebind_scouted_credential_literals
+from skyvern.forge.sdk.copilot.credential_literal_rebind import (
+ rebind_scouted_credential_literals,
+ scouted_credential_targets,
+)
from skyvern.forge.sdk.copilot.credential_pause import preflight_credential_pause
from skyvern.forge.sdk.copilot.data_write_defaults import default_data_write_continue_on_failure
from skyvern.forge.sdk.copilot.enforcement import (
@@ -3893,7 +3896,18 @@ def _inline_replace_workflow_credential_verdict(
if inline_rebind.changed:
workflow_yaml = inline_rebind.workflow_yaml
action_data["workflow_yaml"] = workflow_yaml
- LOG.info("copilot inline REPLACE_WORKFLOW rebound scouted credential literals")
+ LOG.info(
+ "copilot inline REPLACE_WORKFLOW rebound scouted credential literals",
+ credential_literals_rebound=bool(inline_rebind.rebound),
+ authored_credential_fills=[list(pair) for pair in inline_rebind.authored],
+ )
+ for skip in inline_rebind.skips:
+ LOG.info(
+ "copilot inline REPLACE_WORKFLOW credential authoring skipped",
+ stage=skip.stage,
+ selector=skip.selector,
+ targets_present=len(scouted_credential_targets(ctx.scout_trajectory)),
+ )
raw_verdict = evaluate_output_policy(
request_policy=ctx.request_policy,
response_type=resp_type,
@@ -3904,6 +3918,14 @@ def _inline_replace_workflow_credential_verdict(
output_kind=CopilotOutputKind.WORKFLOW_DRAFT_PROPOSAL,
)
hard_block_verdict = hard_block_output_policy_verdict(raw_verdict)
+ if inline_rebind.residual_selectors and hard_block_verdict.allowed:
+ # This path bypasses the update_workflow guardrail, and the output-policy scan does not catch a bare
+ # `page.fill(sel, ...)`, so an inline secret the rebind could not neutralize would slip through here.
+ LOG.info(
+ "copilot inline REPLACE_WORKFLOW credential residual raw fill fail-closed",
+ residual_raw_credential_fill_selectors=list(inline_rebind.residual_selectors),
+ )
+ hard_block_verdict.add(OutputPolicyReason.RAW_SECRET_LEAK)
if not hard_block_verdict.allowed:
_record_output_policy_guardrail_churn(ctx, "replace_workflow_inline", workflow_yaml, hard_block_verdict)
return workflow_yaml, raw_verdict, hard_block_verdict
diff --git a/skyvern/forge/sdk/copilot/code_block_synthesis.py b/skyvern/forge/sdk/copilot/code_block_synthesis.py
index 2d8c22db3..643ab1310 100644
--- a/skyvern/forge/sdk/copilot/code_block_synthesis.py
+++ b/skyvern/forge/sdk/copilot/code_block_synthesis.py
@@ -102,6 +102,92 @@ CREDENTIAL_FILL_CODE_PATTERN = re.compile(r"\.fill\(\s*(?:[A-Za-z_]\w*\.\w+|awai
# Credential fields the scout must fill live before a code block reading them may persist;
# `.otp()` resolves at runtime only, so totp never requires (or credits) a live scout fill.
LIVE_SCOUT_CREDENTIAL_FIELDS = frozenset({"username", "password"})
+
+
+def credential_fill_source(locator_expr: str, param_key: str, field: str) -> str:
+ if field == "totp":
+ return f"await {locator_expr}.fill(await {param_key}.otp())"
+ return f"await {locator_expr}.fill({param_key}.{field})"
+
+
+def wrapped_code_ast(code: str) -> ast.AST | None:
+ body = "\n".join(f" {line}" for line in code.splitlines())
+ if not body.strip():
+ body = " pass"
+ try:
+ return ast.parse(f"async def __submitted_code__():\n{body}\n")
+ except SyntaxError:
+ return None
+
+
+def _credential_field_fill_argument(arg: ast.AST, credential_parameter_keys: AbstractSet[str]) -> bool:
+ if (
+ isinstance(arg, ast.Attribute)
+ and arg.attr in _CREDENTIAL_FIELDS
+ and isinstance(arg.value, ast.Name)
+ and arg.value.id in credential_parameter_keys
+ ):
+ return True
+ target = arg.value if isinstance(arg, ast.Await) else arg
+ return (
+ isinstance(target, ast.Call)
+ and isinstance(target.func, ast.Attribute)
+ and target.func.attr == "otp"
+ and isinstance(target.func.value, ast.Name)
+ and target.func.value.id in credential_parameter_keys
+ )
+
+
+def _is_credential_field_fill_call(node: ast.AST, credential_parameter_keys: AbstractSet[str]) -> bool:
+ return (
+ isinstance(node, ast.Call)
+ and isinstance(node.func, ast.Attribute)
+ and node.func.attr == "fill"
+ and bool(node.args)
+ and _credential_field_fill_argument(node.args[0], credential_parameter_keys)
+ )
+
+
+def _is_presence_guard_test(test: ast.AST) -> bool:
+ for node in ast.walk(test):
+ if isinstance(node, ast.Attribute) and node.attr in {"count", "is_visible"}:
+ return True
+ if isinstance(node, ast.Name) and node.id in _INTERNAL_SCOUT_VARS:
+ return True
+ return False
+
+
+def _credential_fill_is_presence_guarded(node: ast.AST, parents: Mapping[int, ast.AST]) -> bool:
+ current: ast.AST = node
+ while id(current) in parents:
+ parent = parents[id(current)]
+ if (
+ isinstance(parent, ast.If)
+ and any(current is stmt for stmt in parent.body)
+ and _is_presence_guard_test(parent.test)
+ ):
+ return True
+ current = parent
+ return False
+
+
+def block_has_unguarded_credential_fill(code: str, credential_parameter_keys: AbstractSet[str]) -> bool:
+ if not credential_parameter_keys:
+ return False
+ tree = wrapped_code_ast(code)
+ if tree is None:
+ return False
+ parents: dict[int, ast.AST] = {}
+ for node in ast.walk(tree):
+ for child in ast.iter_child_nodes(node):
+ parents[id(child)] = node
+ return any(
+ _is_credential_field_fill_call(node, credential_parameter_keys)
+ and not _credential_fill_is_presence_guarded(node, parents)
+ for node in ast.walk(tree)
+ )
+
+
_CREDENTIAL_FIELD_ACCESS_RE = re.compile(
r"\b(?P<parameter>[A-Za-z_][A-Za-z0-9_]*)\.(?:(?P<field>username|password|totp)\b|(?P<otp_method>otp)\s*\()"
)
@@ -2352,6 +2438,10 @@ def synthesize_code_block(
lines.append(f"{_INDENT}if not {_ENTRY_RESUME_AFTER_AUTH_VAR}:")
lines.append(f"{_INDENT * 2}pass")
if login_only_presence_guard_active:
+ lines.append(f"{_INDENT}try:")
+ lines.append(f'{_INDENT * 2}await {_ENTRY_TARGET_VAR}.wait_for(state="visible", timeout=1000)')
+ lines.append(f"{_INDENT}except Exception:")
+ lines.append(f"{_INDENT * 2}pass")
lines.append(f"{_INDENT}if await {_ENTRY_TARGET_VAR}.count() == 1:")
append_step(f"Open {entry_url}", "goto_url", line_start)
@@ -2593,10 +2683,7 @@ def synthesize_code_block(
credential_param_key = _credential_param_key(interaction, used_param_keys)
credential_param_keys[credential_id] = credential_param_key
parameters.append({"key": credential_param_key, "credential_id": credential_id})
- if credential_field == "totp":
- lines.append(f"{action_indent}await {locator}.fill(await {credential_param_key}.otp())")
- else:
- lines.append(f"{action_indent}await {locator}.fill({credential_param_key}.{credential_field})")
+ lines.append(f"{action_indent}{credential_fill_source(locator, credential_param_key, credential_field)}")
record_emission(trajectory_index, tool_name, "fill", locator, line_start=line_start)
elif tool_name == "select_option":
emit_snapshot_recovery(trajectory_index, action_indent)
diff --git a/skyvern/forge/sdk/copilot/credential_literal_rebind.py b/skyvern/forge/sdk/copilot/credential_literal_rebind.py
index 0dc9831ab..62ebd40dd 100644
--- a/skyvern/forge/sdk/copilot/credential_literal_rebind.py
+++ b/skyvern/forge/sdk/copilot/credential_literal_rebind.py
@@ -1,29 +1,78 @@
from __future__ import annotations
+import ast
import re
+from collections.abc import Set as AbstractSet
from dataclasses import dataclass
from typing import Any, Callable, Mapping, Sequence
import yaml
-from skyvern.forge.sdk.copilot.code_block_synthesis import _CREDENTIAL_FIELDS, CREDENTIAL_FILL_TOOL_NAME
+from skyvern.forge.sdk.copilot.code_block_synthesis import (
+ _CREDENTIAL_FIELDS,
+ CREDENTIAL_FILL_TOOL_NAME,
+ block_has_unguarded_credential_fill,
+ credential_fill_source,
+ wrapped_code_ast,
+)
from skyvern.forge.sdk.copilot.workflow_credential_utils import credential_param_ids, workflow_blocks
+_FILL_METHODS = frozenset({"fill", "type", "press_sequentially"})
+# `wrapped_code_ast` re-indents every line by four spaces, so AST columns are shifted by that much.
+_WRAP_INDENT = 4
+
_IDENTIFIER_RE = re.compile(r"[A-Za-z_]\w*")
_STRING_LITERAL_RE = re.compile(r"(?P<q>['\"])(?:\\.|(?!(?P=q)).)*(?P=q)")
_STRING_ASSIGN_TMPL = r"^[ \t]*{name}[ \t]*=[ \t]*" + _STRING_LITERAL_RE.pattern + r"[ \t]*\n?"
_GOTO_RE = re.compile(r"\.goto\(\s*['\"]([^'\"]+)['\"]")
+_LOCATOR_ALIAS_RE = re.compile(
+ r"^[ \t]*(?P<name>[A-Za-z_]\w*)[ \t]*=[ \t]*page\.locator\(\s*(?P<q>['\"])(?P<sel>(?:\\.|(?!(?P=q)).)*)(?P=q)\s*\)"
+ r"(?:\s*\.\s*(?:first|last))?[ \t]*$",
+ re.MULTILINE,
+)
+
+
+def _locator_alias_bindings(code: str) -> tuple[dict[str, str], set[str]]:
+ """(alias -> selector, conflicted aliases). A name bound to a single literal ``page.locator("<sel>")``
+ maps to that selector; a name rebound to a different selector is ambiguous and dropped into the
+ conflicted set, where a fill through it must fall to the fail-closed backstop rather than be rebound."""
+ aliases: dict[str, str] = {}
+ conflicted: set[str] = set()
+ for match in _LOCATOR_ALIAS_RE.finditer(code):
+ name, selector = match.group("name"), match.group("sel")
+ if name in aliases and aliases[name] != selector:
+ conflicted.add(name)
+ aliases.setdefault(name, selector)
+ for name in conflicted:
+ aliases.pop(name, None)
+ return aliases, conflicted
+
+
+def _locator_alias_selectors(code: str) -> dict[str, str]:
+ return _locator_alias_bindings(code)[0]
def _normalize_page_url(url: str) -> str:
return url.split("#", 1)[0].strip().rstrip("/")
+@dataclass(frozen=True)
+class CredentialRebindSkip:
+ stage: str
+ selector: str
+
+
@dataclass(frozen=True)
class CredentialRebindResult:
workflow_yaml: str
changed: bool
rebound: tuple[str, ...]
+ authored: tuple[tuple[str, str], ...] = ()
+ skips: tuple[CredentialRebindSkip, ...] = ()
+ # A fill at a scouted credential selector still holds a non-parameter value the rewriters could not
+ # neutralize (e.g. a secret inside a nested call). The output-policy secret scan does not catch every
+ # such shape, so the caller must fail closed on this rather than admit the submission.
+ residual_selectors: tuple[str, ...] = ()
def scouted_credential_targets(scout_trajectory: Sequence[Mapping[str, Any]] | None) -> dict[str, tuple[str, str]]:
@@ -110,6 +159,247 @@ def _block_page_incompatible(block_goto_urls: set[str], target_source_urls: froz
return bool(block_goto_urls) and bool(target_source_urls) and not (block_goto_urls & target_source_urls)
+def _fill_value_node(node: ast.Call, positional_index: int) -> ast.AST | None:
+ """The value a fill types: the positional arg at ``positional_index``, else a ``value=`` keyword.
+ Playwright accepts either, and a raw secret passed by keyword must be neutralized just the same."""
+ if len(node.args) > positional_index:
+ return node.args[positional_index]
+ for keyword in node.keywords:
+ if keyword.arg == "value":
+ return keyword.value
+ return None
+
+
+def _fill_call_selector_and_value(
+ node: ast.Call, alias_selectors: Mapping[str, str] | None = None
+) -> tuple[str | None, ast.AST] | None:
+ """For a fill/type/press_sequentially call, return (string-literal selector, value node). The selector
+ is None when the target locator is not a recognizable ``page.fill("<sel>", ...)`` or
+ ``page.locator("<sel>")...`` string-literal shape (e.g. ``get_by_placeholder``), or an alias variable
+ that ``alias_selectors`` cannot resolve back to a literal ``page.locator("<sel>")``. Returns None only
+ when there is no value to inspect at all."""
+ func = node.func
+ if not isinstance(func, ast.Attribute) or func.attr not in _FILL_METHODS:
+ return None
+ receiver = func.value
+ if isinstance(receiver, ast.Name) and receiver.id == "page":
+ first = node.args[0] if node.args else None
+ if isinstance(first, ast.Constant) and isinstance(first.value, str):
+ value = _fill_value_node(node, 1)
+ return (first.value, value) if value is not None else None
+ value = _fill_value_node(node, 1)
+ return (None, value) if value is not None else None
+ base = receiver
+ while isinstance(base, ast.Attribute) and base.attr in {"first", "last"}:
+ base = base.value
+ value = _fill_value_node(node, 0)
+ if value is None:
+ return None
+ if (
+ isinstance(base, ast.Call)
+ and isinstance(base.func, ast.Attribute)
+ and base.func.attr == "locator"
+ and isinstance(base.func.value, ast.Name)
+ and base.func.value.id == "page"
+ and base.args
+ and isinstance(base.args[0], ast.Constant)
+ and isinstance(base.args[0].value, str)
+ ):
+ return base.args[0].value, value
+ if isinstance(base, ast.Name) and alias_selectors:
+ selector = alias_selectors.get(base.id)
+ if selector:
+ return selector, value
+ return None, value
+
+
+def _value_is_param_field(value: ast.AST, param_key: str, field: str) -> bool:
+ if field == "totp":
+ target = value.value if isinstance(value, ast.Await) else value
+ return (
+ isinstance(target, ast.Call)
+ and isinstance(target.func, ast.Attribute)
+ and target.func.attr == "otp"
+ and isinstance(target.func.value, ast.Name)
+ and target.func.value.id == param_key
+ )
+ return (
+ isinstance(value, ast.Attribute)
+ and value.attr == field
+ and isinstance(value.value, ast.Name)
+ and value.value.id == param_key
+ )
+
+
+def _embeds_string_literal(node: ast.AST) -> bool:
+ """True when the expression inlines a non-empty string literal anywhere in its value — directly, in an
+ f-string, or built up via concatenation, ``%``/``.format`` templating, or ``"".join([...])``. These are
+ the shapes that can carry a raw secret. A subscript is a runtime lookup, so its key is not counted; pure
+ runtime references (attribute chains, bare names) inline no literal and are left alone."""
+ target = node.value if isinstance(node, ast.Await) else node
+ if isinstance(target, ast.Constant):
+ return isinstance(target.value, str) and target.value != ""
+ if isinstance(target, ast.Subscript):
+ return _embeds_string_literal(target.value)
+ return any(_embeds_string_literal(child) for child in ast.iter_child_nodes(target))
+
+
+def _value_is_inline_literal_fill(value: ast.AST, code: str) -> bool:
+ """Whether a fill value carries an inline secret that must be neutralized, versus a runtime reference
+ (foreign credential parameter, subscript, identifier bound to a call) that the shipped backstop leaves
+ untouched. Only inline literals — and identifiers bound to one in the same block — are rewritten."""
+ if _embeds_string_literal(value):
+ return True
+ target = value.value if isinstance(value, ast.Await) else value
+ return isinstance(target, ast.Name) and _identifier_bound_to_string_literal(code, target.id)
+
+
+def _standalone_fill_statements(tree: ast.AST) -> dict[int, ast.stmt]:
+ """Map each fill call that IS a whole statement (bare or awaited) to that statement, so only a call
+ whose result nothing consumes may be rewritten wholesale."""
+ standalone: dict[int, ast.stmt] = {}
+ for node in ast.walk(tree):
+ if not isinstance(node, ast.Expr):
+ continue
+ value = node.value
+ if isinstance(value, ast.Await):
+ value = value.value
+ if isinstance(value, ast.Call):
+ standalone[id(value)] = node
+ return standalone
+
+
+def _scan_block_fills(code: str) -> tuple[list[tuple[str, ast.AST, ast.Call, ast.stmt | None]], list[ast.AST]]:
+ """(recognized fills, values of fills whose receiver could not be resolved to a selector). The
+ unrecognized VALUES are returned — not just a flag — because protection keys on the value shape: an
+ inline literal at an unidentifiable receiver must still fail closed, whatever the locator spelling."""
+ tree = wrapped_code_ast(code)
+ if tree is None:
+ return [], []
+ fills: list[tuple[str, ast.AST, ast.Call, ast.stmt | None]] = []
+ unrecognized_values: list[ast.AST] = []
+ alias_selectors = _locator_alias_selectors(code)
+ standalone = _standalone_fill_statements(tree)
+ for node in ast.walk(tree):
+ if not isinstance(node, ast.Call):
+ continue
+ found = _fill_call_selector_and_value(node, alias_selectors)
+ if found is None:
+ continue
+ selector, value = found
+ if selector is None:
+ unrecognized_values.append(value)
+ else:
+ fills.append((selector, value, node, standalone.get(id(node))))
+ return fills, unrecognized_values
+
+
+def _node_source(lines: Sequence[str], node: ast.expr) -> str | None:
+ """Source text of ``node``, undoing the wrapper's line and indent offsets. Multi-line spans are not
+ reconstructed — the caller falls back to the fail-closed skip."""
+ start = node.lineno - 2
+ end = (node.end_lineno or node.lineno) - 2
+ start_col = node.col_offset - _WRAP_INDENT
+ end_col = (node.end_col_offset or 0) - _WRAP_INDENT
+ if start != end or not (0 <= start < len(lines)) or start_col < 0 or end_col < start_col:
+ return None
+ line = lines[start]
+ if end_col > len(line):
+ return None
+ return line[start_col:end_col]
+
+
+def _replace_fill_statement(code: str, stmt: ast.stmt, locator_expr: str, param_key: str, field: str) -> str | None:
+ """Rewrite a whole fill statement into the sanctioned emission, so the raw value is replaced rather
+ than joined by a second fill the secret scan would still reject."""
+ lines = code.splitlines()
+ start = stmt.lineno - 2
+ end = (stmt.end_lineno or stmt.lineno) - 2
+ start_col = stmt.col_offset - _WRAP_INDENT
+ if not (0 <= start <= end < len(lines)) or start_col < 0:
+ return None
+ if lines[start][:start_col].strip():
+ return None
+ indent = lines[start][:start_col]
+ lines[start : end + 1] = [indent + credential_fill_source(locator_expr, param_key, field)]
+ return "\n".join(lines) + ("\n" if code.endswith("\n") else "")
+
+
+def _fill_locator_expr(lines: Sequence[str], call: ast.Call, selector: str) -> str | None:
+ """The locator expression to re-emit the fill against: the draft's own receiver when it has one, else
+ the scouted selector for the two-argument ``page.fill("<sel>", value)`` shape."""
+ func = call.func
+ if not isinstance(func, ast.Attribute):
+ return None
+ if (
+ isinstance(func.value, ast.Name)
+ and func.value.id == "page"
+ and call.args
+ and isinstance(call.args[0], ast.Constant)
+ and isinstance(call.args[0].value, str)
+ ):
+ return f"page.locator({_py_selector(selector)})"
+ return _node_source(lines, func.value)
+
+
+def _has_positive_page_evidence(
+ block_goto_urls: set[str], target_source_urls: frozenset[str], sibling_source_urls: frozenset[str]
+) -> bool:
+ if target_source_urls and block_goto_urls and (block_goto_urls & target_source_urls):
+ return True
+ return bool(target_source_urls) and target_source_urls == sibling_source_urls
+
+
+def _py_selector(selector: str) -> str:
+ return '"' + selector.replace("\\", "\\\\").replace('"', '\\"') + '"'
+
+
+def _author_missing_credential_fill(
+ code: str,
+ selector: str,
+ param_key: str,
+ field: str,
+ sibling_fields: Mapping[str, str],
+ credential_parameter_keys: AbstractSet[str],
+) -> str | None:
+ """Insert a sanctioned credential fill for ``selector`` immediately after a sibling credential fill of
+ the same parameter, matching the sibling's indentation. Returns the new code only when the authored
+ fill is admitted by the persist seam's presence-guard rule, else None."""
+ tree = wrapped_code_ast(code)
+ if tree is None:
+ return None
+ lines = code.splitlines()
+ insert_after: int | None = None
+ indent = ""
+ alias_selectors = _locator_alias_selectors(code)
+ for node in ast.walk(tree):
+ if not isinstance(node, ast.Call):
+ continue
+ found = _fill_call_selector_and_value(node, alias_selectors)
+ if found is None:
+ continue
+ sibling_selector, value = found
+ if sibling_selector is None or sibling_selector not in sibling_fields:
+ continue
+ if not _value_is_param_field(value, param_key, sibling_fields[sibling_selector]):
+ continue
+ start_idx = node.lineno - 2
+ end_idx = (node.end_lineno or node.lineno) - 2
+ if 0 <= start_idx < len(lines):
+ stripped = lines[start_idx].lstrip()
+ indent = lines[start_idx][: len(lines[start_idx]) - len(stripped)]
+ insert_after = end_idx
+ break
+ if insert_after is None:
+ return None
+ authored_line = indent + credential_fill_source(f"page.locator({_py_selector(selector)})", param_key, field)
+ new_lines = lines[: insert_after + 1] + [authored_line] + lines[insert_after + 1 :]
+ new_code = "\n".join(new_lines) + ("\n" if code.endswith("\n") else "")
+ if block_has_unguarded_credential_fill(new_code, credential_parameter_keys):
+ return None
+ return new_code
+
+
def _rebind_block_code(
code: str,
targets: Mapping[str, tuple[str, str]],
@@ -121,6 +411,7 @@ def _rebind_block_code(
replaced_identifiers: set[str] = set()
new_code = code
block_goto_urls = {_normalize_page_url(url) for url in _GOTO_RE.findall(code)}
+ alias_selectors = _locator_alias_selectors(code)
for selector, (credential_id, field) in targets.items():
if _block_page_incompatible(block_goto_urls, source_urls_by_selector.get(selector, frozenset())):
@@ -128,7 +419,7 @@ def _rebind_block_code(
param_key = param_key_for(credential_id)
access = f"{param_key}.{field}"
selector_pattern = re.escape(selector)
- patterns = (
+ patterns = [
re.compile(
r"(\.(?:fill|type|press_sequentially)\(\s*['\"]" + selector_pattern + r"['\"]\s*,\s*)([^),]+?)(\s*\))"
),
@@ -137,6 +428,15 @@ def _rebind_block_code(
+ selector_pattern
+ r"['\"]\s*\)\s*\.(?:fill|type|press_sequentially)\(\s*)([^),]+?)(\s*\))"
),
+ ]
+ patterns.extend(
+ re.compile(
+ r"(\b"
+ + re.escape(alias_name)
+ + r"(?:\s*\.\s*(?:first|last))?\s*\.(?:fill|type|press_sequentially)\(\s*)([^),]+?)(\s*\))"
+ )
+ for alias_name, alias_selector in alias_selectors.items()
+ if alias_selector == selector
)
def _substitute(match: re.Match[str]) -> str:
@@ -164,6 +464,57 @@ def _rebind_block_code(
return new_code, rebound, used_params
+def _residual_raw_credential_fills(
+ blocks: Sequence[Any],
+ targets: Mapping[str, tuple[str, str]],
+ source_urls_by_selector: Mapping[str, frozenset[str]],
+ assigned: Mapping[str, str],
+) -> list[str]:
+ """After rewriting, a fill that still types an inline-literal value at the scouted selector is an
+ un-neutralized secret. A literal fill at a receiver we cannot resolve to a selector at all (``get_by_*``
+ locators, conflicted aliases, arbitrary call chains) counts too — but only while the scouted obligation
+ is UNSATISFIED: a credential typed through an unrecognizable locator necessarily leaves its sanctioned
+ fill missing, whereas a draft whose credential fills are all sanctioned may legitimately fill a search
+ box or date field with a literal, and rejecting that would refuse a correct attempt (the disease this
+ module exists to cure). The net keys on the VALUE shape, never on recognizing the receiver; the
+ output-policy scan misses the bare ``.fill(...)`` shapes, so the caller must fail closed on these."""
+ residual: list[str] = []
+ for selector, (credential_id, field) in targets.items():
+ param_key = assigned.get(credential_id)
+ if param_key is None:
+ continue
+ target_source_urls = source_urls_by_selector.get(selector, frozenset())
+ satisfied = False
+ flagged = False
+ for block in blocks:
+ if not isinstance(block, dict):
+ continue
+ code = block.get("code")
+ if not isinstance(code, str) or not code.strip():
+ continue
+ if _block_page_incompatible({_normalize_page_url(u) for u in _GOTO_RE.findall(code)}, target_source_urls):
+ continue
+ fills, unrecognized_values = _scan_block_fills(code)
+ if any(fsel == selector and _value_is_param_field(fval, param_key, field) for fsel, fval, _n, _s in fills):
+ satisfied = True
+ recognized_literal = any(
+ fsel == selector
+ and _value_is_inline_literal_fill(fval, code)
+ and not _value_is_param_field(fval, param_key, field)
+ for fsel, fval, _n, _s in fills
+ )
+ unrecognized_literal = any(_value_is_inline_literal_fill(value, code) for value in unrecognized_values)
+ if recognized_literal:
+ residual.append(selector)
+ flagged = True
+ break
+ if unrecognized_literal:
+ flagged = True
+ if flagged and not satisfied and selector not in residual:
+ residual.append(selector)
+ return residual
+
+
def rebind_scouted_credential_literals(
workflow_yaml: str | None, scout_trajectory: Sequence[Mapping[str, Any]] | None
) -> CredentialRebindResult:
@@ -179,19 +530,28 @@ def rebind_scouted_credential_literals(
targets = scouted_credential_targets(scout_trajectory)
if not targets:
return empty
+
+ def _skipped(stage: str) -> CredentialRebindResult:
+ return CredentialRebindResult(
+ workflow_yaml=workflow_yaml or "",
+ changed=False,
+ rebound=(),
+ skips=tuple(CredentialRebindSkip(stage, selector) for selector in targets),
+ )
+
source_urls_by_selector = scouted_selector_source_urls(scout_trajectory)
try:
parsed = yaml.safe_load(workflow_yaml)
except yaml.YAMLError:
- return empty
+ return _skipped("yaml-parse")
if not isinstance(parsed, dict):
- return empty
+ return _skipped("no-definition")
workflow_definition = parsed.get("workflow_definition")
if not isinstance(workflow_definition, dict):
- return empty
+ return _skipped("no-definition")
blocks = workflow_blocks(parsed)
if not blocks:
- return empty
+ return _skipped("no-blocks")
parameters = workflow_definition.get("parameters")
parameters = parameters if isinstance(parameters, list) else []
@@ -227,8 +587,24 @@ def rebind_scouted_credential_literals(
block["parameter_keys"] = parameter_keys
minted_for.update(used_params)
- if not all_rebound:
- return empty
+ for credential_id, _field in targets.values():
+ param_key_for(credential_id)
+ credential_keys: set[str] = set(credential_param_ids(parameters)) | set(assigned.values())
+
+ authored, skips = _author_missing_obligations(
+ blocks, targets, source_urls_by_selector, assigned, minted_for, credential_keys
+ )
+
+ residual = tuple(_residual_raw_credential_fills(blocks, targets, source_urls_by_selector, assigned))
+
+ if not all_rebound and not authored:
+ return CredentialRebindResult(
+ workflow_yaml=workflow_yaml or "",
+ changed=False,
+ rebound=(),
+ skips=tuple(skips),
+ residual_selectors=residual,
+ )
for credential_id, key in assigned.items():
if key not in minted_for:
@@ -242,4 +618,145 @@ def rebind_scouted_credential_literals(
workflow_yaml=yaml.safe_dump(parsed, sort_keys=False),
changed=True,
rebound=tuple(all_rebound),
+ authored=tuple(authored),
+ skips=tuple(skips),
+ residual_selectors=residual,
)
+
+
+def _author_missing_obligations(
+ blocks: Sequence[Any],
+ targets: Mapping[str, tuple[str, str]],
+ source_urls_by_selector: Mapping[str, frozenset[str]],
+ assigned: Mapping[str, str],
+ minted_for: set[str],
+ credential_keys: AbstractSet[str],
+) -> tuple[list[tuple[str, str]], list[CredentialRebindSkip]]:
+ authored: list[tuple[str, str]] = []
+ skips: list[CredentialRebindSkip] = []
+
+ def block_infos() -> list[
+ tuple[dict[str, Any], set[str], list[tuple[str, ast.AST, ast.Call, ast.stmt | None]], bool]
+ ]:
+ infos = []
+ for block in blocks:
+ if not isinstance(block, dict):
+ continue
+ code = block.get("code")
+ if not isinstance(code, str) or not code.strip():
+ continue
+ goto = {_normalize_page_url(url) for url in _GOTO_RE.findall(code)}
+ fills, unrecognized_values = _scan_block_fills(code)
+ infos.append((block, goto, fills, bool(unrecognized_values)))
+ return infos
+
+ for selector, (credential_id, field) in targets.items():
+ param_key = assigned[credential_id]
+ target_source_urls = source_urls_by_selector.get(selector, frozenset())
+ satisfied = False
+ unmatched: tuple[dict[str, Any], ast.Call, ast.stmt, ast.AST] | None = None
+ inline_literal_present = False
+ runtime_reference_present = False
+ saw_compatible_block = False
+ for block, goto, fills, _unrecognized in block_infos():
+ if _block_page_incompatible(goto, target_source_urls):
+ continue
+ saw_compatible_block = True
+ block_code = str(block.get("code") or "")
+ for fsel, fval, fnode, fstmt in fills:
+ if fsel != selector:
+ continue
+ if _value_is_param_field(fval, param_key, field):
+ satisfied = True
+ elif _value_is_inline_literal_fill(fval, block_code):
+ inline_literal_present = True
+ if unmatched is None and fstmt is not None:
+ unmatched = (block, fnode, fstmt, fval)
+ else:
+ runtime_reference_present = True
+ if satisfied:
+ continue
+ if not saw_compatible_block:
+ skips.append(CredentialRebindSkip("page-incompatible", selector))
+ continue
+ if runtime_reference_present and not inline_literal_present:
+ # The selector is already filled with a runtime reference (a foreign credential parameter, a
+ # subscript, an identifier bound to a call). It carries no inline secret; leave it as the
+ # shipped backstop does rather than author a duplicate or clobber the author's value.
+ continue
+ if inline_literal_present:
+ # Replace the draft's own credential fill with the sanctioned reference. The secret scan does
+ # NOT catch the bare `page.fill(sel, "...")` shape, so keeping the raw value would leak it; an
+ # unguarded parameter fill is strictly safer (the persist seam refuses an unguarded fill, but a
+ # refused draft never persists a secret). Guardedness is not a reason to keep the literal.
+ if unmatched is not None:
+ block, call, stmt, old_value = unmatched
+ code = str(block.get("code") or "")
+ locator_expr = _fill_locator_expr(code.splitlines(), call, selector)
+ new_code = _replace_fill_statement(code, stmt, locator_expr, param_key, field) if locator_expr else None
+ if new_code is not None:
+ # The replaced value may have been an identifier bound to a literal; the now-dead
+ # `var = "<secret>"` assignment would keep the raw credential in the block.
+ old_target = old_value.value if isinstance(old_value, ast.Await) else old_value
+ if isinstance(old_target, ast.Name):
+ new_code = _drop_dead_string_assignment(new_code, old_target.id)
+ block["code"] = new_code
+ parameter_keys = block.get("parameter_keys")
+ parameter_keys = list(parameter_keys) if isinstance(parameter_keys, list) else []
+ if param_key not in parameter_keys:
+ parameter_keys.append(param_key)
+ block["parameter_keys"] = parameter_keys
+ minted_for.add(param_key)
+ authored.append((selector, field))
+ continue
+ skips.append(CredentialRebindSkip("literal-unmatched", selector))
+ continue
+
+ sibling_fields = {
+ sib_selector: sib_field
+ for sib_selector, (sib_credential_id, sib_field) in targets.items()
+ if sib_selector != selector and sib_credential_id == credential_id
+ }
+ anchor = None
+ for block, goto, fills, has_unrecognized in block_infos():
+ if _block_page_incompatible(goto, target_source_urls):
+ continue
+ sibling = next(
+ (
+ (fsel, fval)
+ for fsel, fval, _fnode, _fstmt in fills
+ if fsel in sibling_fields and _value_is_param_field(fval, param_key, sibling_fields[fsel])
+ ),
+ None,
+ )
+ if sibling is not None:
+ anchor = (block, goto, has_unrecognized, sibling[0])
+ break
+ if anchor is None:
+ skips.append(CredentialRebindSkip("no-anchor", selector))
+ continue
+ anchor_block, anchor_goto, anchor_has_unrecognized, sibling_selector = anchor
+ if anchor_has_unrecognized:
+ skips.append(CredentialRebindSkip("unrecognized-fill-shape", selector))
+ continue
+ if not _has_positive_page_evidence(
+ anchor_goto, target_source_urls, source_urls_by_selector.get(sibling_selector, frozenset())
+ ):
+ skips.append(CredentialRebindSkip("no-page-evidence", selector))
+ continue
+ new_code = _author_missing_credential_fill(
+ str(anchor_block.get("code") or ""), selector, param_key, field, sibling_fields, credential_keys
+ )
+ if new_code is None:
+ skips.append(CredentialRebindSkip("guard-inadmissible", selector))
+ continue
+ anchor_block["code"] = new_code
+ parameter_keys = anchor_block.get("parameter_keys")
+ parameter_keys = list(parameter_keys) if isinstance(parameter_keys, list) else []
+ if param_key not in parameter_keys:
+ parameter_keys.append(param_key)
+ anchor_block["parameter_keys"] = parameter_keys
+ minted_for.add(param_key)
+ authored.append((selector, field))
+
+ return authored, skips
diff --git a/skyvern/forge/sdk/copilot/tools/guardrails.py b/skyvern/forge/sdk/copilot/tools/guardrails.py
index 692658b43..2b3047a08 100644
--- a/skyvern/forge/sdk/copilot/tools/guardrails.py
+++ b/skyvern/forge/sdk/copilot/tools/guardrails.py
@@ -19,7 +19,11 @@ from skyvern.forge.sdk.copilot.composition_evidence import (
workflow_target_url,
)
from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
-from skyvern.forge.sdk.copilot.credential_literal_rebind import rebind_scouted_credential_literals
+from skyvern.forge.sdk.copilot.credential_literal_rebind import (
+ CredentialRebindResult,
+ rebind_scouted_credential_literals,
+ scouted_credential_targets,
+)
from skyvern.forge.sdk.copilot.enforcement import _record_code_authoring_guardrail_reject
from skyvern.forge.sdk.copilot.loop_detection import record_consecutive_tool_result_boundary_for_ctx
from skyvern.forge.sdk.copilot.output_policy import (
@@ -82,8 +86,10 @@ def _workflow_yaml_output_policy_guardrail(data: ToolInputGuardrailData) -> Tool
workflow_yaml_value = tool_arguments.get("workflow_yaml")
workflow_yaml = workflow_yaml_value if isinstance(workflow_yaml_value, str) else None
- rebound_yaml = _rebind_scouted_credential_literals_in_place(tool_context, tool_arguments, workflow_yaml)
- effective_yaml = rebound_yaml if rebound_yaml is not None else workflow_yaml
+ rebind_result = _rebind_scouted_credential_literals_in_place(tool_context, tool_arguments, workflow_yaml)
+ applied = rebind_result is not None and rebind_result.changed
+ effective_yaml = rebind_result.workflow_yaml if rebind_result is not None and applied else workflow_yaml
+ _log_credential_authoring_skips(tool_context, rebind_result)
verdict = evaluate_output_policy(
request_policy=getattr(getattr(tool_context, "context", None), "request_policy", None),
@@ -95,9 +101,37 @@ def _workflow_yaml_output_policy_guardrail(data: ToolInputGuardrailData) -> Tool
surface="tool_input",
tool_name=getattr(tool_context, "tool_name", None),
)
+ residual_selectors = rebind_result.residual_selectors if rebind_result is not None else ()
+ if verdict.allowed and residual_selectors:
+ # A scouted credential selector still holds a non-parameter value the rebind could not neutralize,
+ # and the output-policy scan does not catch every such shape. Fail closed rather than persist it.
+ trace_data = {
+ **trace_data,
+ "allowed": False,
+ "residual_raw_credential_fill_selectors": list(residual_selectors),
+ }
+ LOG.info("copilot output policy credential residual raw fill fail-closed", **trace_data)
+ error = (
+ "A credential value could not be safely bound to a workflow parameter for selector(s) "
+ f"{', '.join(residual_selectors)}. Fill these fields with `<credential_key>.username` / "
+ "`<credential_key>.password` (or `await <credential_key>.otp()`) rather than an inline value."
+ )
+ tool_name = getattr(tool_context, "tool_name", None)
+ if isinstance(tool_name, str) and tool_name:
+ record_consecutive_tool_result_boundary_for_ctx(
+ getattr(tool_context, "context", None),
+ tool_name,
+ {"ok": False, "error": error},
+ arguments=tool_arguments,
+ )
+ return ToolGuardrailFunctionOutput.reject_content(error, output_info=trace_data)
if verdict.allowed:
- if rebound_yaml is not None:
- trace_data = {**trace_data, "credential_literals_rebound": True}
+ if applied and rebind_result is not None:
+ trace_data = {
+ **trace_data,
+ "credential_literals_rebound": True,
+ "authored_credential_fills": [list(pair) for pair in rebind_result.authored],
+ }
LOG.info("copilot output policy rebound scouted credential literals", **trace_data)
else:
LOG.info("copilot output policy tool guardrail verdict", **trace_data)
@@ -119,22 +153,21 @@ def _workflow_yaml_output_policy_guardrail(data: ToolInputGuardrailData) -> Tool
def _rebind_scouted_credential_literals_in_place(
tool_context: Any, tool_arguments: dict[str, Any], workflow_yaml: str | None
-) -> str | None:
- """Rewrite scouted credential literals to parameter access before the output policy runs. A
- non-deferred function tool (as update_workflow is today) is invoked by the SDK from
- ``tool_context.tool_call.arguments``, so the rebound YAML must be written there and into
- ``tool_arguments``; returns None (leaving the original YAML for the clamp to fail closed on) when
- nothing was rebound, no tool_call is present, or the write-back fails."""
+) -> CredentialRebindResult | None:
+ """Rewrite scouted credential literals to parameter access and author missing sanctioned credential
+ fills into ``tool_context.tool_call.arguments`` before the output policy runs. Returns the full result
+ (None only without an ``AgentContext``); a changed result is downgraded to ``changed=False`` when no
+ tool_call is present or the write-back fails, so the clamp fails closed while skips still surface."""
ctx = getattr(tool_context, "context", None)
if not isinstance(ctx, AgentContext):
return None
result = rebind_scouted_credential_literals(workflow_yaml, ctx.scout_trajectory)
if not result.changed:
- return None
+ return result
tool_call = getattr(tool_context, "tool_call", None)
if tool_call is None:
LOG.warning("copilot rebound credential yaml has no tool_call to persist through; failing closed")
- return None
+ return _unapplied_rebind_result(result, workflow_yaml)
tool_arguments["workflow_yaml"] = result.workflow_yaml
try:
rebound_arguments = json.dumps(tool_arguments)
@@ -143,8 +176,36 @@ def _rebind_scouted_credential_literals_in_place(
except (AttributeError, TypeError, ValueError):
LOG.warning("copilot rebound credential yaml could not be written back to tool arguments")
tool_arguments["workflow_yaml"] = workflow_yaml
- return None
- return result.workflow_yaml
+ return _unapplied_rebind_result(result, workflow_yaml)
+ return result
+
+
+def _unapplied_rebind_result(result: CredentialRebindResult, workflow_yaml: str | None) -> CredentialRebindResult:
+ return CredentialRebindResult(
+ workflow_yaml=workflow_yaml or "",
+ changed=False,
+ rebound=(),
+ skips=result.skips,
+ residual_selectors=result.residual_selectors,
+ )
+
+
+def _log_credential_authoring_skips(tool_context: Any, result: CredentialRebindResult | None) -> None:
+ if result is None or not result.skips:
+ return
+ ctx = getattr(tool_context, "context", None)
+ targets_present = (
+ len(scouted_credential_targets(ctx.scout_trajectory)) if isinstance(ctx, AgentContext) else len(result.skips)
+ )
+ tool_name = getattr(tool_context, "tool_name", None)
+ for skip in result.skips:
+ LOG.info(
+ "copilot output policy credential authoring skipped",
+ tool_name=tool_name,
+ stage=skip.stage,
+ selector=skip.selector,
+ targets_present=targets_present,
+ )
def _record_output_policy_guardrail_churn(
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index dc5e1f61f..4eaa1d399 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -10,7 +10,6 @@ import re
import textwrap
import tokenize
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
-from collections.abc import Set as AbstractSet
from dataclasses import dataclass
from dataclasses import field as dataclass_field
from dataclasses import replace
@@ -86,7 +85,6 @@ from skyvern.forge.sdk.copilot.code_block_security import CodeBlockSecurityError
from skyvern.forge.sdk.copilot.code_block_steps import apply_derived_code_block_steps, fill_code_block_prompts_in_yaml
from skyvern.forge.sdk.copilot.code_block_synthesis import (
_CODE_SUBMIT_ACTION_RE,
- _CREDENTIAL_FIELDS,
_INTERNAL_SCOUT_VARS,
_SYNTHESIZED_BLOCK_LABEL,
CREDENTIAL_FILL_TOOL_NAME,
@@ -111,6 +109,11 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import (
_selector_refines,
artifact_dependency_id,
artifact_observation_ref_id,
+)
+from skyvern.forge.sdk.copilot.code_block_synthesis import (
+ block_has_unguarded_credential_fill as _block_has_unguarded_credential_fill,
+)
+from skyvern.forge.sdk.copilot.code_block_synthesis import (
build_input_templated_locator,
build_same_month_file_match_locator,
credential_scout_gap,
@@ -138,6 +141,7 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import (
uncovered_rung_records,
validated_dynamic_row_period_matches,
)
+from skyvern.forge.sdk.copilot.code_block_synthesis import wrapped_code_ast as _wrapped_code_ast
from skyvern.forge.sdk.copilot.completion_verification import grade_definition_criteria
from skyvern.forge.sdk.copilot.composition_evidence import (
SCOUT_INTERACTION_EVIDENCE_TOOL,
@@ -10328,74 +10332,6 @@ def _workflow_credential_parameter_keys(parsed: Mapping[str, Any]) -> set[str]:
return keys
-def _credential_field_fill_argument(arg: ast.AST, credential_parameter_keys: AbstractSet[str]) -> bool:
- if (
- isinstance(arg, ast.Attribute)
- and arg.attr in _CREDENTIAL_FIELDS
- and isinstance(arg.value, ast.Name)
- and arg.value.id in credential_parameter_keys
- ):
- return True
- target = arg.value if isinstance(arg, ast.Await) else arg
- return (
- isinstance(target, ast.Call)
- and isinstance(target.func, ast.Attribute)
- and target.func.attr == "otp"
- and isinstance(target.func.value, ast.Name)
- and target.func.value.id in credential_parameter_keys
- )
-
-
-def _is_credential_field_fill_call(node: ast.AST, credential_parameter_keys: AbstractSet[str]) -> bool:
- return (
- isinstance(node, ast.Call)
- and isinstance(node.func, ast.Attribute)
- and node.func.attr == "fill"
- and bool(node.args)
- and _credential_field_fill_argument(node.args[0], credential_parameter_keys)
- )
-
-
-def _is_presence_guard_test(test: ast.AST) -> bool:
- for node in ast.walk(test):
- if isinstance(node, ast.Attribute) and node.attr in {"count", "is_visible"}:
- return True
- if isinstance(node, ast.Name) and node.id in _INTERNAL_SCOUT_VARS:
- return True
- return False
-
-
-def _credential_fill_is_presence_guarded(node: ast.AST, parents: Mapping[int, ast.AST]) -> bool:
- current: ast.AST = node
- while id(current) in parents:
- parent = parents[id(current)]
- if (
- isinstance(parent, ast.If)
- and any(current is stmt for stmt in parent.body)
- and _is_presence_guard_test(parent.test)
- ):
- return True
- current = parent
- return False
-
-
-def _block_has_unguarded_credential_fill(code: str, credential_parameter_keys: AbstractSet[str]) -> bool:
- if not credential_parameter_keys:
- return False
- tree = _wrapped_code_ast(code)
- if tree is None:
- return False
- parents: dict[int, ast.AST] = {}
- for node in ast.walk(tree):
- for child in ast.iter_child_nodes(node):
- parents[id(child)] = node
- return any(
- _is_credential_field_fill_call(node, credential_parameter_keys)
- and not _credential_fill_is_presence_guarded(node, parents)
- for node in ast.walk(tree)
- )
-
-
def _freehand_surface_reject(
workflow_yaml: str, code_blocks: list[dict[str, Any]], validation: _BrowserSurfaceValidation
) -> _SynthesizedCodeImpositionResult:
@@ -10591,16 +10527,6 @@ def _is_submitted_code_synthesized_only(submitted_code: str, synthesized_code: s
return bool(submitted and synthesized and submitted == synthesized)
-def _wrapped_code_ast(code: str) -> ast.AST | None:
- body = "\n".join(f" {line}" for line in code.splitlines())
- if not body.strip():
- body = " pass"
- try:
- return ast.parse(f"async def __submitted_code__():\n{body}\n")
- except SyntaxError:
- return None
-
-
def _is_page_locator_expression(value: ast.AST) -> bool:
# Peel a trailing `.first`/`.last` (the synthesizer's disambiguator for a bare role/tag selector)
# so literal-fill parameter binding still recognizes the underlying page locator.
diff --git a/tests/unit/test_copilot_code_block_synthesis.py b/tests/unit/test_copilot_code_block_synthesis.py
index 1ff912c3a..b9d3fe151 100644
--- a/tests/unit/test_copilot_code_block_synthesis.py
+++ b/tests/unit/test_copilot_code_block_synthesis.py
@@ -74,6 +74,7 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import (
synthesize_extraction_suffix,
templated_selection_locator_binding,
uncovered_required_emitted_interactions,
+ uncovered_rung_records,
witness_prelude_lines,
)
from skyvern.forge.sdk.copilot.context import (
@@ -91,7 +92,7 @@ from skyvern.forge.sdk.copilot.reached_download_target import ReachedDownloadTar
from skyvern.forge.sdk.copilot.run_outcome import run_outcome_display_reason
from skyvern.forge.sdk.copilot.tools import _normalize_code_artifact_metadata
from skyvern.forge.sdk.copilot.tools.scouting import _fill_carry_to_interaction, _with_trajectory_anchor
-from skyvern.forge.sdk.copilot.tools.workflow_update import _code_block_safety_errors
+from skyvern.forge.sdk.copilot.tools.workflow_update import _code_block_safety_errors, _scouted_spine_omission_digest
from skyvern.forge.sdk.workflow.exceptions import CustomizedCodeException
from skyvern.forge.sdk.workflow.models.block import CodeBlock, CodeBlockStep
@@ -2774,6 +2775,19 @@ class TestCredentialFillSynthesis:
def test_runtime_otp_fill_is_detected_as_credential_fill_code(self) -> None:
assert code_contains_credential_fill('await page.locator("#otp").fill(await login_credential.otp())')
+ def test_demonstrated_totp_is_a_required_rung_flagged_when_a_draft_omits_it(self) -> None:
+ trajectory = [
+ self._credential_fill(),
+ self._credential_fill(selector="#totpCode", credential_field="totp", typed_length=6),
+ ]
+ result = synthesize_code_block(trajectory, strict_selectors=True)
+ assert result is not None
+ emitted = result.diagnostics.emitted_interactions
+ assert all(not str(record.get("lane") or "") for record in emitted)
+ draft_omitting_totp = [("fill", 'page.locator("#userName")')]
+ uncovered = uncovered_required_emitted_interactions(emitted, draft_omitting_totp)
+ assert [record.get("selector") for record in uncovered] == ["#totpCode"]
+
def test_missing_credential_reference_is_dropped_with_note(self) -> None:
result = synthesize_code_block(
[
@@ -3937,6 +3951,33 @@ class TestSpinePartitionFindings:
draft_calls = _covering_draft_calls(result.diagnostics)
assert spine_partition_findings(result.diagnostics, draft_calls, self._spine_trajectory()) == []
+ def test_omitted_demonstrated_totp_fill_is_an_uncovered_rung_finding(self) -> None:
+ login_url = "https://example.com/login"
+ trajectory = [
+ _interaction(
+ CREDENTIAL_FILL_TOOL_NAME,
+ selector="#userName",
+ source_url=login_url,
+ credential_id="cred_1",
+ credential_field="username",
+ typed_length=20,
+ ),
+ _interaction(
+ CREDENTIAL_FILL_TOOL_NAME,
+ selector="#totpCode",
+ source_url=login_url,
+ credential_id="cred_1",
+ credential_field="totp",
+ typed_length=6,
+ ),
+ ]
+ result = synthesize_code_block(trajectory, strict_selectors=True)
+ assert result is not None
+ draft_calls = [("fill", 'page.locator("#userName")')]
+ findings = spine_partition_findings(result.diagnostics, draft_calls, trajectory)
+ uncovered = uncovered_rung_records(findings)
+ assert [record.get("selector") for record in uncovered] == ["#totpCode"]
+
def test_unforgiven_drop_is_a_typed_finding(self) -> None:
trajectory = [
{"tool_name": "click", "selector": "#stage-a", "source_url": "https://example.com/records"},
@@ -4766,6 +4807,9 @@ class _FakeLocator:
return self._page.counts.get(self._selector, 0)
async def wait_for(self, *, state: str, timeout: float | None = None) -> None:
+ if self._page.goto_done and self._selector in self._page.appears_after_goto:
+ self._page.counts.update(self._page.appears_after_goto)
+ return
if self._page.counts.get(self._selector, 0) < 1:
raise TimeoutError(f"{self._selector} not visible")
@@ -4784,18 +4828,21 @@ class _FakeLocator:
class _FakePage:
- def __init__(self, counts: dict[str, int]) -> None:
+ def __init__(self, counts: dict[str, int], appears_after_goto: dict[str, int] | None = None) -> None:
self.counts = counts
self.filled: list[str] = []
self.clicked: list[str] = []
self.pressed: list[tuple[str, str]] = []
self.goto_calls: list[str] = []
+ self.appears_after_goto = appears_after_goto or {}
+ self.goto_done = False
def locator(self, selector: str) -> _FakeLocator:
return _FakeLocator(self, selector)
async def goto(self, url: str, *, wait_until: str | None = None) -> None:
self.goto_calls.append(url)
+ self.goto_done = True
async def wait_for_load_state(self, state: str) -> None:
return None
@@ -4872,6 +4919,18 @@ class TestLoginOnlyPresenceGuardSynthesis:
assert page.filled == []
assert page.clicked == []
+ def test_login_field_rendered_after_goto_is_filled_not_skipped(self) -> None:
+ traj = self._login_only_trajectory(
+ submit=_interaction("click", selector="#login-btn", source_url="https://example.com/login")
+ )
+ result = synthesize_code_block(traj, strict_selectors=True)
+ assert result is not None
+ page = _FakePage({}, appears_after_goto={"#username": 1, "#password": 1, "#login-btn": 1})
+ _run_synthesized_block(result.code, page, SimpleNamespace(username="u", password="p"))
+ assert page.filled == ["#username", "#password"]
+ assert page.clicked == ["#login-btn"]
+ assert page.goto_calls == ["https://example.com/login"]
+
class TestSharedSubmitPredicate:
def test_click_and_enter_are_submits_but_tab_is_not(self) -> None:
@@ -4944,3 +5003,23 @@ def test_type_text_secret_bypass_is_not_carried_into_synthesized_block() -> None
assert _INLINE_SECRET_SENTINEL not in result.code
credential_param = next(param for param in result.parameters if param.get("credential_id") == "cred_x")
assert f"{credential_param['key']}.username" in result.code
+
+
+class TestScoutedSpineOmissionDigest:
+ @staticmethod
+ def _record(selector: str, index: int = 1) -> dict[str, Any]:
+ return {"tool_name": "click", "method": "click", "selector": selector, "trajectory_index": index}
+
+ def test_identical_omissions_share_a_digest_regardless_of_order(self) -> None:
+ forward = _scouted_spine_omission_digest([self._record("#a", 1), self._record("#b", 2)])
+ reversed_order = _scouted_spine_omission_digest([self._record("#b", 2), self._record("#a", 1)])
+ assert forward == reversed_order
+
+ def test_distinct_omissions_produce_distinct_digests(self) -> None:
+ assert _scouted_spine_omission_digest([self._record("#a")]) != _scouted_spine_omission_digest(
+ [self._record("#b")]
+ )
+
+ def test_digest_is_cross_process_stable_sha256_not_salted_hash(self) -> None:
+ digest = _scouted_spine_omission_digest([self._record("#search-submit", 0)])
+ assert digest == "8065b147a155c4e35cab8b3b35da9beab958cddc9c355b6200bac957878954ec"