Skip to content

Automate browser based workflows with AI

SKY-12415 Keep date-driven row selection stable as lists change (#7686)

Andrew Neilson7bc491369e0b
diff --git a/skyvern/config.py b/skyvern/config.py
index 2bee911e2..752b7ab5b 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -108,6 +108,7 @@ class Settings(BaseSettings):
TEMP_PATH: str = "./temp"
DOWNLOAD_PATH: str = f"{REPO_ROOT_DIR}/downloads"
BROWSER_ACTION_TIMEOUT_MS: int = 5000
+ BROWSER_ACTION_MAX_EXECUTION_SECONDS: int = 1200
POPUP_VIDEO_PATH_TIMEOUT_SECONDS: float = 3.0
CACHED_ACTION_DELAY_SECONDS: float = 1.0
# Page readiness settings for cached action execution
diff --git a/skyvern/exceptions.py b/skyvern/exceptions.py
index 1446f7af5..dac3cf702 100644
--- a/skyvern/exceptions.py
+++ b/skyvern/exceptions.py
@@ -1112,6 +1112,15 @@ class InvalidWorkflowParameter(SkyvernHTTPException):
)
+class ActionExecutionTimeout(SkyvernException):
+ def __init__(self, action_type: str, timeout_seconds: float):
+ super().__init__(
+ f"Action execution timed out after {timeout_seconds:.0f} seconds and was aborted"
+ f" (action_type={action_type}). The browser action did not complete in time —"
+ " the page or browser may have become unresponsive."
+ )
+
+
class InteractWithDisabledElement(SkyvernException):
def __init__(self, element_id: str):
super().__init__(
diff --git a/skyvern/forge/sdk/copilot/code_block_synthesis.py b/skyvern/forge/sdk/copilot/code_block_synthesis.py
index 2809ef4ac..2d8c22db3 100644
--- a/skyvern/forge/sdk/copilot/code_block_synthesis.py
+++ b/skyvern/forge/sdk/copilot/code_block_synthesis.py
@@ -10,6 +10,7 @@ from __future__ import annotations
import ast
import hashlib
+import hmac
import io
import json
import keyword
@@ -24,6 +25,7 @@ from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
import structlog
+from skyvern.config import settings
from skyvern.forge.sdk.copilot.authoring_parameter_binding import (
AuthoringParameterBindingSnapshot,
SameMonthFileMatchTransform,
@@ -41,7 +43,12 @@ from skyvern.forge.sdk.copilot.output_extraction_plan import (
)
from skyvern.forge.sdk.copilot.reached_download_target import ReachedDownloadTarget
from skyvern.forge.sdk.copilot.request_slots import is_canonical_request_slot_path
-from skyvern.forge.sdk.copilot.runtime import ScoutedInputCorrespondence
+from skyvern.forge.sdk.copilot.runtime import (
+ ScoutedDynamicRowEvidence,
+ ScoutedDynamicRowPeriodMatch,
+ ScoutedEquivalentInput,
+ ScoutedInputCorrespondence,
+)
from skyvern.utils.strings import escape_code_fences
LOG = structlog.get_logger()
@@ -59,6 +66,10 @@ _ENTRY_OPENER_VAR = "_scout_entry_opener"
_OPTIONAL_DISMISSAL_VAR = "_scout_optional_dismissal"
_READONLY_DEFERRED_VAR = "_scout_readonly_actual"
_MONTH_HELPER_VAR = "_scout_month_to_iso"
+_ISO_DATE_HELPER_VAR = "_scout_iso_date_to_year_month"
+_PERIOD_DATE_PATTERN_HELPER_VAR = "_scout_period_date_pattern"
+_DYNAMIC_ROW_EVIDENCE_FINGERPRINT_DOMAIN = b"skyvern.copilot.dynamic_row_evidence.v1"
+_DYNAMIC_ROW_EVIDENCE_SCRYPT_N = 1 << 14
_ENTRY_LOCATOR_VARS = (_ENTRY_TARGET_VAR, _ENTRY_RESUME_TARGET_VAR, _ENTRY_OPENER_VAR)
_INTERNAL_SCOUT_VARS = (
_ENTRY_TARGET_VAR,
@@ -71,6 +82,8 @@ _INTERNAL_SCOUT_VARS = (
_READONLY_DEFERRED_VAR,
_MONTH_HELPER_VAR,
_SAME_MONTH_HELPER_VAR,
+ _ISO_DATE_HELPER_VAR,
+ _PERIOD_DATE_PATTERN_HELPER_VAR,
)
# Base name for the download var bound by `async with page.expect_download() as <name>:`.
@@ -272,6 +285,8 @@ _RESERVED_PARAM_NAMES = frozenset(
_ENTRY_RESUME_TARGET_VAR,
_ENTRY_OPENER_VAR,
_MONTH_HELPER_VAR,
+ _ISO_DATE_HELPER_VAR,
+ _PERIOD_DATE_PATTERN_HELPER_VAR,
_DOWNLOAD_VAR_BASE,
f"{_DOWNLOAD_VAR_BASE}_file",
_DOWNLOAD_FILENAME_VAR_BASE,
@@ -308,6 +323,8 @@ class SynthesisDiagnostics:
# Post-download-cut trajectory indices recorded before the emission loop, so the partition obligation
# can detect a truncation-break index that lands in no record lane instead of silently losing it.
retained_trajectory_indices: list[int] = field(default_factory=list)
+ # May contain bounded source-row text for in-memory fingerprint validation. The only public boundary
+ # is `_public_locator_provenance`, which emits origin/input metadata and omits the captured text.
locator_provenance: list[dict[str, Any]] = field(default_factory=list)
# (trajectory enumerate index -> minted type_text parameter key); diagnostics-only, never serialized.
# Recovers the key for a typed field whose value was withheld from default_value (typed_value == "").
@@ -584,6 +601,18 @@ _WITNESS_MONTH_TO_ISO = {
"november": "11",
"december": "12",
}
+_PERIOD_DAY_PATTERN_BY_MAX = {
+ 28: r"(?:0?[1-9]|1[0-9]|2[0-8])",
+ 29: r"(?:0?[1-9]|1[0-9]|2[0-9])",
+ 30: r"(?:0?[1-9]|[12][0-9]|30)",
+ 31: r"(?:0?[1-9]|[12][0-9]|3[01])",
+}
+# Intentionally scoped to the English "Month D, YYYY" labels this grounded route can witness.
+# Additional formats must preserve Python/browser parity instead of adding website-specific parsing.
+_ROW_PERIOD_DATE_RE = re.compile(
+ r"\b(" + "|".join(_WITNESS_MONTH_TO_ISO) + r")\s+(0?[1-9]|[12][0-9]|3[01]),\s+([0-9]{4})\b",
+ re.IGNORECASE,
+)
class _InputTemplatingPlan(NamedTuple):
@@ -592,6 +621,7 @@ class _InputTemplatingPlan(NamedTuple):
role: str
name: str
holes: list[Mapping[str, Any]]
+ dynamic_row_evidence: ScoutedDynamicRowEvidence | None
def _witness_key_is_safe(key: str) -> bool:
@@ -615,14 +645,231 @@ def _month_name_to_iso(value: str) -> str | None:
return f"{year}-{month}"
+def _days_in_month(year: int, month: int) -> int:
+ if month == 2:
+ return 29 if year % 4 == 0 and (year % 100 != 0 or year % 400 == 0) else 28
+ return 30 if month in (4, 6, 9, 11) else 31
+
+
+def _iso_date_to_year_month(value: str) -> str | None:
+ parts = value.split("-")
+ if len(parts) != 3:
+ return None
+ year_text, month_text, day_text = parts
+ if (
+ len(year_text) != 4
+ or len(month_text) != 2
+ or len(day_text) != 2
+ or not year_text.isdigit()
+ or not month_text.isdigit()
+ or not day_text.isdigit()
+ ):
+ return None
+ year, month, day = int(year_text), int(month_text), int(day_text)
+ if year < 1 or month < 1 or month > 12 or day < 1 or day > _days_in_month(year, month):
+ return None
+ return f"{year_text}-{month_text}"
+
+
def _witness_observed_forms(value: str) -> list[tuple[str, str]]:
forms: list[tuple[str, str]] = [("identity", value)]
iso = _month_name_to_iso(value)
if iso is not None and iso != value:
forms.append(("month_name_to_iso", iso))
+ year_month = _iso_date_to_year_month(value)
+ if year_month is not None and year_month != value:
+ forms.append(("iso_date_to_year_month", year_month))
return forms
+def _row_period_tokens(row_text: str) -> list[tuple[str, int]]:
+ normalized = " ".join(row_text.split())
+ periods: list[tuple[str, int]] = []
+ for match in _ROW_PERIOD_DATE_RE.finditer(normalized):
+ month_name = match.group(1).lower()
+ month = _WITNESS_MONTH_TO_ISO.get(month_name)
+ if month is None:
+ continue
+ day = int(match.group(2))
+ year = match.group(3)
+ year_number = int(year)
+ if year_number < 1 or day < 1 or day > _days_in_month(year_number, int(month)):
+ continue
+ periods.append((f"{year}-{month}", match.start()))
+ return periods
+
+
+def _strict_period_date_pattern(period: str) -> re.Pattern[str] | None:
+ parts = period.split("-")
+ if (
+ len(parts) != 2
+ or len(parts[0]) != 4
+ or not parts[0].isdigit()
+ or int(parts[0]) < 1
+ or len(parts[1]) != 2
+ or not parts[1].isdigit()
+ or not 1 <= int(parts[1]) <= 12
+ ):
+ return None
+ month_names = tuple(name.title() for name in _WITNESS_MONTH_TO_ISO)
+ max_day = _days_in_month(int(parts[0]), int(parts[1]))
+ day = _PERIOD_DAY_PATTERN_BY_MAX[max_day]
+ return re.compile(
+ rf"\b{re.escape(month_names[int(parts[1]) - 1])}\s+{day},\s+{re.escape(parts[0])}\b",
+ re.IGNORECASE,
+ )
+
+
+def validated_dynamic_row_period_matches(
+ value: Any, row_selector_count: int
+) -> list[ScoutedDynamicRowPeriodMatch] | None:
+ if not isinstance(value, list) or len(value) > 20:
+ return None
+ result: list[ScoutedDynamicRowPeriodMatch] = []
+ for item in value:
+ if not isinstance(item, Mapping) or set(item) != {"period", "selected_row_match_count", "row_match_count"}:
+ return None
+ period = item.get("period")
+ selected_count = item.get("selected_row_match_count")
+ row_count = item.get("row_match_count")
+ if (
+ not isinstance(period, str)
+ or _strict_period_date_pattern(period) is None
+ or isinstance(selected_count, bool)
+ or not isinstance(selected_count, int)
+ or selected_count < 1
+ or selected_count > 20
+ or isinstance(row_count, bool)
+ or not isinstance(row_count, int)
+ or row_count < 1
+ or row_count > row_selector_count
+ ):
+ return None
+ result.append(
+ ScoutedDynamicRowPeriodMatch(
+ period=period,
+ selected_row_match_count=selected_count,
+ row_match_count=row_count,
+ )
+ )
+ if [item["period"] for item in result] != sorted({str(item["period"]) for item in result}):
+ return None
+ return result
+
+
+def dynamic_row_period_matches_match_selected_row(row_text: str, period_matches: Sequence[Mapping[str, Any]]) -> bool:
+ selected_counts: dict[str, int] = {}
+ for period, _ in _row_period_tokens(row_text):
+ selected_counts[period] = selected_counts.get(period, 0) + 1
+ return selected_counts == {str(item["period"]): int(item["selected_row_match_count"]) for item in period_matches}
+
+
+def dynamic_row_evidence_fingerprint(
+ *,
+ source_url: str,
+ target_selector: str,
+ row_selector: str,
+ row_text: str,
+ row_selector_count: int,
+ row_text_match_count: int,
+ period_matches: Sequence[Mapping[str, Any]],
+ selected_index: int,
+) -> str:
+ """Return a keyed integrity tag for potentially sensitive captured row evidence."""
+ payload = {
+ "source_url": source_url,
+ "target_selector": target_selector,
+ "row_selector": row_selector,
+ "row_text": row_text,
+ "row_selector_count": row_selector_count,
+ "row_text_match_count": row_text_match_count,
+ "period_matches": [dict(item) for item in period_matches],
+ "selected_index": selected_index,
+ }
+ serialized = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()
+ return hashlib.scrypt(
+ serialized,
+ salt=_DYNAMIC_ROW_EVIDENCE_FINGERPRINT_DOMAIN + b"\x00" + settings.SECRET_KEY.encode(),
+ n=_DYNAMIC_ROW_EVIDENCE_SCRYPT_N,
+ r=8,
+ p=1,
+ dklen=32,
+ ).hex()
+
+
+def _validated_dynamic_row_evidence(interaction: Mapping[str, Any]) -> ScoutedDynamicRowEvidence | None:
+ evidence = interaction.get("dynamic_row_evidence")
+ if not isinstance(evidence, Mapping):
+ return None
+ source_url = str(interaction.get("source_url") or "").strip()
+ selector = str(interaction.get("selector") or "").strip()
+ row_source_url = evidence.get("source_url")
+ target_selector = evidence.get("target_selector")
+ row_selector = evidence.get("row_selector")
+ row_text = evidence.get("row_text")
+ row_selector_count = evidence.get("row_selector_count")
+ row_text_match_count = evidence.get("row_text_match_count")
+ period_matches = (
+ validated_dynamic_row_period_matches(evidence.get("period_matches"), row_selector_count)
+ if isinstance(row_selector_count, int) and not isinstance(row_selector_count, bool)
+ else None
+ )
+ selected_index = evidence.get("selected_index")
+ evidence_fingerprint = evidence.get("evidence_fingerprint")
+ if (
+ not source_url
+ or row_source_url != source_url
+ or target_selector != selector
+ or not isinstance(row_selector, str)
+ or not row_selector.strip()
+ or _is_positional_selector(row_selector)
+ or _is_bare_ambiguous_selector(row_selector)
+ or not isinstance(row_text, str)
+ or not row_text.strip()
+ or len(row_text) > 500
+ or isinstance(row_selector_count, bool)
+ or not isinstance(row_selector_count, int)
+ or row_selector_count < 2
+ or row_selector_count > 100
+ or isinstance(row_text_match_count, bool)
+ or not isinstance(row_text_match_count, int)
+ or row_text_match_count < 1
+ or row_text_match_count > row_selector_count
+ or period_matches is None
+ or not dynamic_row_period_matches_match_selected_row(" ".join(row_text.split()), period_matches)
+ or isinstance(selected_index, bool)
+ or not isinstance(selected_index, int)
+ or selected_index < 0
+ or selected_index >= row_selector_count
+ or not isinstance(evidence_fingerprint, str)
+ or not hmac.compare_digest(
+ evidence_fingerprint,
+ dynamic_row_evidence_fingerprint(
+ source_url=source_url,
+ target_selector=selector,
+ row_selector=row_selector.strip(),
+ row_text=" ".join(row_text.split()),
+ row_selector_count=row_selector_count,
+ row_text_match_count=row_text_match_count,
+ period_matches=period_matches,
+ selected_index=selected_index,
+ ),
+ )
+ ):
+ return None
+ return ScoutedDynamicRowEvidence(
+ source_url=source_url,
+ target_selector=selector,
+ row_selector=row_selector.strip(),
+ row_text=" ".join(row_text.split()),
+ row_selector_count=row_selector_count,
+ row_text_match_count=row_text_match_count,
+ period_matches=period_matches,
+ selected_index=selected_index,
+ evidence_fingerprint=evidence_fingerprint,
+ )
+
+
def _quoted_content_spans(selector: str) -> list[tuple[int, int]]:
spans: list[tuple[int, int]] = []
quote = ""
@@ -666,36 +913,61 @@ def _boundary_delimited_positions(haystack: str, needle: str, allowed_spans: Seq
def _resolve_non_competing_correspondences(raw: list[dict[str, Any]]) -> list[ScoutedInputCorrespondence]:
result: list[ScoutedInputCorrespondence] = []
- for surface in ("selector", "accessible_name"):
- entries = sorted((entry for entry in raw if entry["surface"] == surface), key=lambda entry: entry["_position"])
- key_counts: dict[str, int] = {}
+ for surface in ("selector", "accessible_name", "row_text"):
+ entries = sorted(
+ (entry for entry in raw if entry["surface"] == surface),
+ key=lambda entry: (entry["_position"], entry["matched_literal"], entry["input_key"]),
+ )
+ key_counts: dict[tuple[str, str, str], int] = {}
for entry in entries:
- key_counts[entry["input_key"]] = key_counts.get(entry["input_key"], 0) + 1
+ occurrence = (entry["input_key"], entry["matched_literal"], entry["transform"])
+ key_counts[occurrence] = key_counts.get(occurrence, 0) + 1
+ groups: list[list[dict[str, Any]]] = []
+ for entry in entries:
+ if groups and (groups[-1][0]["_position"], groups[-1][0]["matched_literal"]) == (
+ entry["_position"],
+ entry["matched_literal"],
+ ):
+ groups[-1].append(entry)
+ else:
+ groups.append([entry])
bad: set[int] = set()
- for a_index, a_entry in enumerate(entries):
- if key_counts[a_entry["input_key"]] > 1:
+ for a_index, a_group in enumerate(groups):
+ if any(
+ key_counts[(entry["input_key"], entry["matched_literal"], entry["transform"])] > 1 for entry in a_group
+ ):
bad.add(a_index)
- a_start = a_entry["_position"]
- a_end = a_start + len(a_entry["matched_literal"])
- for b_index in range(a_index + 1, len(entries)):
- b_start = entries[b_index]["_position"]
- b_end = b_start + len(entries[b_index]["matched_literal"])
+ a_start = a_group[0]["_position"]
+ a_end = a_start + len(a_group[0]["matched_literal"])
+ for b_index in range(a_index + 1, len(groups)):
+ b_start = groups[b_index][0]["_position"]
+ b_end = b_start + len(groups[b_index][0]["matched_literal"])
if a_start < b_end and b_start < a_end:
bad.add(a_index)
bad.add(b_index)
- for index, entry in enumerate(entries):
+ for index, group in enumerate(groups):
if index in bad:
continue
- result.append(
- {
- "input_key": entry["input_key"],
- "matched_literal": entry["matched_literal"],
- "parameter_value": entry["parameter_value"],
- "surface": entry["surface"],
- "transform": entry["transform"],
- "position": entry["_position"],
- }
+ ordered = sorted(group, key=lambda entry: entry["input_key"])
+ canonical = ordered[0]
+ correspondence = ScoutedInputCorrespondence(
+ input_key=canonical["input_key"],
+ matched_literal=canonical["matched_literal"],
+ parameter_value=canonical["parameter_value"],
+ surface=canonical["surface"],
+ transform=canonical["transform"],
+ position=canonical["_position"],
)
+ if len(ordered) > 1:
+ correspondence["equivalent_inputs"] = [
+ ScoutedEquivalentInput(
+ input_key=entry["input_key"],
+ parameter_value=entry["parameter_value"],
+ transform=entry["transform"],
+ )
+ for entry in ordered[1:]
+ ]
+ result.append(correspondence)
return result
@@ -704,9 +976,16 @@ def _input_correspondences_for_surfaces(
selector: str,
name: str,
declared_params: Mapping[str, str],
+ dynamic_row: ScoutedDynamicRowEvidence | None = None,
) -> list[ScoutedInputCorrespondence]:
selector_spans = _quoted_content_spans(selector)
name_spans = [(0, len(name))] if name else []
+ row_periods = _row_period_tokens(dynamic_row["row_text"]) if dynamic_row is not None else []
+ licensed_periods = {
+ str(item["period"])
+ for item in (dynamic_row["period_matches"] if dynamic_row is not None else [])
+ if item["selected_row_match_count"] == 1 and item["row_match_count"] == 1
+ }
raw: list[dict[str, Any]] = []
for key in sorted(declared_params):
value = declared_params[key]
@@ -716,27 +995,69 @@ def _input_correspondences_for_surfaces(
continue
if not _witness_key_is_safe(key):
continue
+ identity_selector_positions = _boundary_delimited_positions(selector, value, selector_spans)
+ identity_name_positions = _boundary_delimited_positions(name, value, name_spans)
for transform, observed in _witness_observed_forms(value):
if len(observed) < _WITNESS_MIN_VALUE_LEN or not _WITNESS_SAFE_CHARSET_RE.fullmatch(observed):
continue
- selector_positions = _boundary_delimited_positions(selector, observed, selector_spans)
- name_positions = _boundary_delimited_positions(name, observed, name_spans)
- if len(selector_positions) + len(name_positions) != 1:
- continue
- if selector_positions:
- surface, position = "selector", selector_positions[0]
- else:
- surface, position = "accessible_name", name_positions[0]
- raw.append(
- {
- "surface": surface,
- "input_key": key,
- "matched_literal": observed,
- "parameter_value": value,
- "transform": transform,
- "_position": position,
- }
+ selector_positions = (
+ identity_selector_positions
+ if transform == "identity"
+ else _boundary_delimited_positions(selector, observed, selector_spans)
+ )
+ name_positions = (
+ identity_name_positions
+ if transform == "identity"
+ else _boundary_delimited_positions(name, observed, name_spans)
)
+ if transform != "identity":
+ selector_positions = [
+ position
+ for position in selector_positions
+ if not any(
+ exact_position <= position and position + len(observed) <= exact_position + len(value)
+ for exact_position in identity_selector_positions
+ )
+ ]
+ name_positions = [
+ position
+ for position in name_positions
+ if not any(
+ exact_position <= position and position + len(observed) <= exact_position + len(value)
+ for exact_position in identity_name_positions
+ )
+ ]
+ if len(selector_positions) + len(name_positions) == 1:
+ if selector_positions:
+ surface, position = "selector", selector_positions[0]
+ else:
+ surface, position = "accessible_name", name_positions[0]
+ raw.append(
+ {
+ "surface": surface,
+ "input_key": key,
+ "matched_literal": observed,
+ "parameter_value": value,
+ "transform": transform,
+ "_position": position,
+ }
+ )
+ matching_row_periods = [
+ (period, position)
+ for period, position in row_periods
+ if period == observed and period in licensed_periods
+ ]
+ if len(matching_row_periods) == 1:
+ raw.append(
+ {
+ "surface": "row_text",
+ "input_key": key,
+ "matched_literal": observed,
+ "parameter_value": value,
+ "transform": transform,
+ "_position": matching_row_periods[0][1],
+ }
+ )
return _resolve_non_competing_correspondences(raw)
@@ -764,6 +1085,7 @@ def input_correspondences_for_interaction(
selector=str(interaction.get("selector") or "").strip(),
name=str(interaction.get("accessible_name") or "").strip(),
declared_params=declared_params,
+ dynamic_row=_validated_dynamic_row_evidence(interaction),
)
@@ -787,21 +1109,54 @@ def _interpolate_holes(raw: str, holes: Sequence[Mapping[str, Any]]) -> str | No
return None
segments.append(_escape_fstring_literal_segment(raw[cursor:idx]))
key = str(hole.get("input_key") or "")
- if str(hole.get("transform") or "identity") == "month_name_to_iso":
- segments.append("{" + _SCOUT_MONTH_HELPER_NAME + "(" + key + ")}")
- else:
- segments.append("{" + key + "}")
+ expression = _witness_transform_expression(key, str(hole.get("transform") or "identity"))
+ if expression is None:
+ return None
+ segments.append("{" + expression + "}")
cursor = idx + len(matched_literal)
segments.append(_escape_fstring_literal_segment(raw[cursor:]))
return "".join(segments)
+def _witness_transform_expression(key: str, transform: str) -> str | None:
+ if transform == "identity":
+ return key
+ if transform == "month_name_to_iso":
+ return f"{_SCOUT_MONTH_HELPER_NAME}({key})"
+ if transform == "iso_date_to_year_month":
+ return f"{_ISO_DATE_HELPER_VAR}({key})"
+ return None
+
+
+def _input_templated_holes_are_self_validating(holes: Sequence[Mapping[str, Any]]) -> bool:
+ for hole in holes:
+ matched_literal = str(hole.get("matched_literal") or "")
+ inputs = _correspondence_inputs(hole)
+ keys = [witness["input_key"] for witness in inputs]
+ if not matched_literal or not keys or keys != sorted(set(keys)):
+ return False
+ for witness in inputs:
+ if not _witness_key_is_safe(witness["input_key"]):
+ return False
+ forms = _witness_observed_forms(witness["parameter_value"])
+ if (witness["transform"], matched_literal) not in forms:
+ return False
+ return True
+
+
def build_input_templated_locator(
- *, surface: str, selector: str, role: str, name: str, holes: Sequence[Mapping[str, Any]]
+ *,
+ surface: str,
+ selector: str,
+ role: str,
+ name: str,
+ holes: Sequence[Mapping[str, Any]],
+ row_text: str = "",
+ period_matches: Sequence[Mapping[str, Any]] = (),
) -> str | None:
"""Single source for the templated locator literal, used at emission AND re-derived byte-for-byte at
the admissibility seam so a tampered or reordered provenance record fails the recompute equality check."""
- if not holes:
+ if not holes or not _input_templated_holes_are_self_validating(holes):
return None
if surface == "selector":
body = _interpolate_holes(selector, holes)
@@ -815,6 +1170,27 @@ def build_input_templated_locator(
if body is None:
return None
return f'page.get_by_role({_py_str(role)}, name=f"{body}", exact=True)'
+ if surface == "row_text":
+ if len(holes) != 1 or not selector or not row_text:
+ return None
+ hole = holes[0]
+ matched_literal = str(hole.get("matched_literal") or "")
+ if (
+ len([period for period, _ in _row_period_tokens(row_text) if period == matched_literal]) != 1
+ or _strict_period_date_pattern(matched_literal) is None
+ or not any(
+ item.get("period") == matched_literal
+ and item.get("selected_row_match_count") == 1
+ and item.get("row_match_count") == 1
+ for item in period_matches
+ )
+ ):
+ return None
+ key = str(hole.get("input_key") or "")
+ transformed = _witness_transform_expression(key, str(hole.get("transform") or "identity"))
+ if transformed is None:
+ return None
+ return f"page.locator({_py_str(selector)}).filter(has_text={_PERIOD_DATE_PATTERN_HELPER_VAR}({transformed}))"
return None
@@ -830,7 +1206,13 @@ def templated_selection_locator_binding(interaction: Mapping[str, Any]) -> tuple
if not key:
return None
expr = build_input_templated_locator(
- surface=plan.surface, selector=plan.selector, role=plan.role, name=plan.name, holes=plan.holes
+ surface=plan.surface,
+ selector=plan.selector,
+ role=plan.role,
+ name=plan.name,
+ holes=plan.holes,
+ row_text=plan.dynamic_row_evidence["row_text"] if plan.dynamic_row_evidence is not None else "",
+ period_matches=plan.dynamic_row_evidence["period_matches"] if plan.dynamic_row_evidence is not None else (),
)
if expr is None:
return None
@@ -871,7 +1253,18 @@ def _input_templating_plan(interaction: Mapping[str, Any]) -> _InputTemplatingPl
name = str(interaction.get("accessible_name") or "").strip()
selector_holes = [c for c in correspondences if isinstance(c, Mapping) and c.get("surface") == "selector"]
name_holes = [c for c in correspondences if isinstance(c, Mapping) and c.get("surface") == "accessible_name"]
+ row_holes = [c for c in correspondences if isinstance(c, Mapping) and c.get("surface") == "row_text"]
parsed = _parse_role_name(selector) if selector else None
+ dynamic_row = _validated_dynamic_row_evidence(interaction)
+ if row_holes and dynamic_row is not None and len(row_holes) == 1:
+ return _InputTemplatingPlan(
+ surface="row_text",
+ selector=dynamic_row["row_selector"],
+ role="",
+ name="",
+ holes=row_holes,
+ dynamic_row_evidence=dynamic_row,
+ )
if (
selector_holes
and selector
@@ -881,13 +1274,27 @@ def _input_templating_plan(interaction: Mapping[str, Any]) -> _InputTemplatingPl
):
ordered = _ordered_holes(selector, selector_holes)
if ordered is not None:
- return _InputTemplatingPlan(surface="selector", selector=selector, role="", name="", holes=ordered)
+ return _InputTemplatingPlan(
+ surface="selector",
+ selector=selector,
+ role="",
+ name="",
+ holes=ordered,
+ dynamic_row_evidence=None,
+ )
if name_holes and role and name:
ambiguous_role = parsed is not None and not parsed[1]
if not selector or _is_bare_ambiguous_selector(selector) or ambiguous_role:
ordered = _ordered_holes(name, name_holes)
if ordered is not None:
- return _InputTemplatingPlan(surface="accessible_name", selector="", role=role, name=name, holes=ordered)
+ return _InputTemplatingPlan(
+ surface="accessible_name",
+ selector="",
+ role=role,
+ name=name,
+ holes=ordered,
+ dynamic_row_evidence=None,
+ )
return None
@@ -901,7 +1308,13 @@ def _maybe_input_templated_locator(
if plan is None:
return None
expr = build_input_templated_locator(
- surface=plan.surface, selector=plan.selector, role=plan.role, name=plan.name, holes=plan.holes
+ surface=plan.surface,
+ selector=plan.selector,
+ role=plan.role,
+ name=plan.name,
+ holes=plan.holes,
+ row_text=plan.dynamic_row_evidence["row_text"] if plan.dynamic_row_evidence is not None else "",
+ period_matches=plan.dynamic_row_evidence["period_matches"] if plan.dynamic_row_evidence is not None else (),
)
if expr is None:
return None
@@ -918,33 +1331,76 @@ def _maybe_input_templated_locator(
"parameter_value": str(hole.get("parameter_value") or ""),
"transform": str(hole.get("transform") or "identity"),
"position": hole.get("position"),
+ **(
+ {"equivalent_inputs": [dict(equivalent) for equivalent in hole["equivalent_inputs"]]}
+ if isinstance(hole.get("equivalent_inputs"), list)
+ else {}
+ ),
}
for hole in plan.holes
],
}
if plan.surface == "selector":
record["selector"] = plan.selector
- else:
+ elif plan.surface == "accessible_name":
record["role"] = plan.role
record["name"] = plan.name
+ elif plan.dynamic_row_evidence is not None:
+ record.update(plan.dynamic_row_evidence)
diagnostics.locator_provenance.append(record)
return expr
-def _prescan_input_templating(trajectory: Sequence[Mapping[str, Any]]) -> tuple[list[str], bool]:
+def _correspondence_inputs(hole: Mapping[str, Any]) -> list[dict[str, str]]:
+ inputs = [
+ {
+ "input_key": str(hole.get("input_key") or ""),
+ "parameter_value": str(hole.get("parameter_value") or ""),
+ "transform": str(hole.get("transform") or "identity"),
+ }
+ ]
+ equivalents = hole.get("equivalent_inputs")
+ if isinstance(equivalents, list):
+ for equivalent in equivalents:
+ if not isinstance(equivalent, Mapping):
+ continue
+ inputs.append(
+ {
+ "input_key": str(equivalent.get("input_key") or ""),
+ "parameter_value": str(equivalent.get("parameter_value") or ""),
+ "transform": str(equivalent.get("transform") or "identity"),
+ }
+ )
+ return inputs
+
+
+def _prescan_input_templating(
+ trajectory: Sequence[Mapping[str, Any]],
+) -> tuple[list[str], bool, bool, bool, list[list[dict[str, str]]]]:
keys: list[str] = []
needs_month = False
+ needs_iso_date = False
+ needs_period_helpers = False
+ collision_groups: list[list[dict[str, str]]] = []
for interaction in trajectory:
plan = _input_templating_plan(interaction)
if plan is None:
continue
+ if plan.surface == "row_text":
+ needs_period_helpers = True
for hole in plan.holes:
- key = str(hole.get("input_key") or "")
- if key and key not in keys:
- keys.append(key)
- if str(hole.get("transform") or "identity") == "month_name_to_iso":
- needs_month = True
- return keys, needs_month
+ inputs = _correspondence_inputs(hole)
+ if len(inputs) > 1:
+ collision_groups.append(inputs)
+ for witness in inputs:
+ key = witness["input_key"]
+ if key and key not in keys:
+ keys.append(key)
+ if witness["transform"] == "month_name_to_iso":
+ needs_month = True
+ elif witness["transform"] == "iso_date_to_year_month":
+ needs_iso_date = True
+ return keys, needs_month, needs_iso_date, needs_period_helpers, collision_groups
def _scout_month_helper_lines() -> list[str]:
@@ -960,6 +1416,42 @@ def _scout_month_helper_lines() -> list[str]:
]
+def _scout_iso_date_helper_lines() -> list[str]:
+ return [
+ f"{_INDENT}def {_ISO_DATE_HELPER_VAR}(_value):",
+ f"{_INDENT * 2}_parts = str(_value).split('-')",
+ f"{_INDENT * 2}if not (len(_parts) == 3 and len(_parts[0]) == 4 and len(_parts[1]) == 2 "
+ f"and len(_parts[2]) == 2 and all(_part.isdigit() for _part in _parts)):",
+ f'{_INDENT * 3}raise Exception("unrecognized ISO date for grounded parameter")',
+ f"{_INDENT * 2}_year, _month, _day = int(_parts[0]), int(_parts[1]), int(_parts[2])",
+ f"{_INDENT * 2}_leap = _year % 4 == 0 and (_year % 100 != 0 or _year % 400 == 0)",
+ f"{_INDENT * 2}_days = (31, 29 if _leap else 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31)",
+ f"{_INDENT * 2}if _year < 1 or _month < 1 or _month > 12 or _day < 1 or _day > _days[_month - 1]:",
+ f'{_INDENT * 3}raise Exception("unrecognized ISO date for grounded parameter")',
+ f"{_INDENT * 2}return _parts[0] + '-' + _parts[1]",
+ ]
+
+
+def _scout_period_helper_lines() -> list[str]:
+ month_names = "(" + ", ".join(f'"{name.title()}"' for name in _WITNESS_MONTH_TO_ISO) + ")"
+ day_patterns = repr(_PERIOD_DAY_PATTERN_BY_MAX)
+ return [
+ f"{_INDENT}def {_PERIOD_DATE_PATTERN_HELPER_VAR}(_period):",
+ f"{_INDENT * 2}_parts = str(_period).split('-')",
+ f"{_INDENT * 2}if not (len(_parts) == 2 and len(_parts[0]) == 4 and _parts[0].isdigit() "
+ f"and len(_parts[1]) == 2 and _parts[1].isdigit() and int(_parts[0]) >= 1 "
+ f"and 1 <= int(_parts[1]) <= 12):",
+ f'{_INDENT * 3}raise Exception("unrecognized grounded period")',
+ f"{_INDENT * 2}_months = {month_names}",
+ f"{_INDENT * 2}_year, _month = int(_parts[0]), int(_parts[1])",
+ f"{_INDENT * 2}_leap = _year % 4 == 0 and (_year % 100 != 0 or _year % 400 == 0)",
+ f"{_INDENT * 2}_max_day = (31, 29 if _leap else 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31)[_month - 1]",
+ f"{_INDENT * 2}_day = {day_patterns}[_max_day]",
+ f'{_INDENT * 2}return re.compile(r"\\b" + re.escape(_months[int(_parts[1]) - 1]) + r"\\s+" '
+ f'+ _day + r",\\s+" + re.escape(_parts[0]) + r"\\b", re.IGNORECASE)',
+ ]
+
+
def _witness_charset_guard_lines(key: str) -> list[str]:
return [
f"{_INDENT}if not (isinstance({key}, str) and {key} == {key}.strip() and {key}[:1].isalnum() "
@@ -968,14 +1460,36 @@ def _witness_charset_guard_lines(key: str) -> list[str]:
]
-def witness_prelude_lines(keys: Sequence[str], *, include_month_helper: bool) -> list[str]:
+def witness_prelude_lines(
+ keys: Sequence[str],
+ *,
+ include_month_helper: bool,
+ include_iso_date_helper: bool = False,
+ include_period_helpers: bool = False,
+ collision_groups: Sequence[Sequence[Mapping[str, str]]] = (),
+) -> list[str]:
"""Top-of-body guards (fail closed before any interpolation) plus the reserved month helper def.
Reinjected into every separated browser stage because each stage is an independent CodeBlock."""
lines: list[str] = []
if include_month_helper:
lines.extend(_scout_month_helper_lines())
+ if include_iso_date_helper:
+ lines.extend(_scout_iso_date_helper_lines())
+ if include_period_helpers:
+ lines.extend(_scout_period_helper_lines())
for key in keys:
lines.extend(_witness_charset_guard_lines(key))
+ for group in collision_groups:
+ transformed = [_witness_transform_expression(witness["input_key"], witness["transform"]) for witness in group]
+ if not transformed or any(expression is None for expression in transformed):
+ continue
+ canonical = transformed[0]
+ peers = transformed[1:]
+ if not peers:
+ continue
+ comparison = " and ".join(f"{canonical} == {peer}" for peer in peers)
+ lines.append(f"{_INDENT}if not ({comparison}):")
+ lines.append(f'{_INDENT * 2}raise Exception("grounded parameters do not resolve to one period")')
return lines
@@ -1071,6 +1585,27 @@ def _locator_expr(
if templated is not None:
return templated
+ correspondences = interaction.get("input_correspondences")
+ has_stamped_row_license = isinstance(correspondences, list) and any(
+ isinstance(item, Mapping) and item.get("surface") == "row_text" for item in correspondences
+ )
+
+ if strict_selectors and (
+ has_stamped_row_license
+ or ("dynamic_row_evidence" in interaction and _validated_dynamic_row_evidence(interaction) is None)
+ ):
+ notes.append("dropped an interaction whose dynamic-row relation did not validate")
+ if diagnostics is not None:
+ diagnostics.dropped_interactions.append(
+ {
+ "trajectory_index": trajectory_index if trajectory_index is not None else -1,
+ "tool_name": tool_name,
+ "selector": selector,
+ "reason_code": "invalid_dynamic_row_evidence",
+ }
+ )
+ return ""
+
if strict_selectors:
if not selector:
notes.append("dropped an interaction with no selector")
@@ -1540,22 +2075,29 @@ def synthesize_code_block(
)
diagnostics.retained_trajectory_indices = list(range(len(trajectory)))
- input_templated_keys, input_templated_needs_month = _prescan_input_templating(trajectory)
+ (
+ input_templated_keys,
+ input_templated_needs_month,
+ input_templated_needs_iso_date,
+ input_templated_needs_period_helpers,
+ input_templated_collision_groups,
+ ) = _prescan_input_templating(trajectory)
minted_input_witness_keys: set[str] = set()
for interaction in trajectory:
plan = _input_templating_plan(interaction)
if plan is None:
continue
- for plan_hole in plan.holes:
- key = str(plan_hole.get("input_key") or "")
- if not key or key in minted_input_witness_keys:
- continue
- minted_input_witness_keys.add(key)
- parameter = {"key": key, "source": LOCATOR_WITNESS_PARAM_SOURCE}
- # A file-match witness proves usage, not a safe persisted default; the submitted declaration owns it.
- if key not in file_match_keys:
- parameter["default_value"] = str(plan_hole.get("parameter_value") or "")
- parameters.append(parameter)
+ for hole in plan.holes:
+ for witness in _correspondence_inputs(hole):
+ key = witness["input_key"]
+ if not key or key in minted_input_witness_keys:
+ continue
+ minted_input_witness_keys.add(key)
+ parameter = {"key": key, "source": LOCATOR_WITNESS_PARAM_SOURCE}
+ # A file-match witness proves usage, not a safe persisted default; the submitted declaration owns it.
+ if key not in file_match_keys:
+ parameter["default_value"] = witness["parameter_value"]
+ parameters.append(parameter)
for key in input_templated_keys:
used_param_keys.add(key)
for key in file_match_keys:
@@ -1564,11 +2106,21 @@ def synthesize_code_block(
parameters.append({"key": key})
prelude_keys = [*input_templated_keys, *(key for key in file_match_keys if key not in input_templated_keys)]
if prelude_keys:
- lines.extend(witness_prelude_lines(prelude_keys, include_month_helper=input_templated_needs_month))
+ lines.extend(
+ witness_prelude_lines(
+ prelude_keys,
+ include_month_helper=input_templated_needs_month,
+ include_iso_date_helper=input_templated_needs_iso_date,
+ include_period_helpers=input_templated_needs_period_helpers,
+ collision_groups=input_templated_collision_groups,
+ )
+ )
LOG.info(
"copilot_spine_input_templated_prelude",
witness_keys=prelude_keys,
month_helper=input_templated_needs_month,
+ iso_date_helper=input_templated_needs_iso_date,
+ period_helpers=input_templated_needs_period_helpers,
)
if file_match_locator and file_match_transform is not None:
lines.extend(_same_month_helper_lines())
@@ -1957,6 +2509,13 @@ def synthesize_code_block(
lane="optional_dismissal",
)
else:
+ templating_plan = _input_templating_plan(interaction)
+ if templating_plan is not None and templating_plan.surface == "row_text":
+ lines.append(f"{action_indent}if await {locator}.count() != 1:")
+ lines.append(
+ f"{action_indent}{_INDENT}raise Exception("
+ f"{_py_str('grounded statement row did not resolve uniquely')})"
+ )
lines.append(f"{action_indent}await {locator}.click()")
lines.append(f"{action_indent}await page.wait_for_load_state({_py_str(_DOMCONTENTLOADED)})")
record_emission(trajectory_index, tool_name, "click", locator, line_start=line_start)
diff --git a/skyvern/forge/sdk/copilot/composition_browser_expressions.py b/skyvern/forge/sdk/copilot/composition_browser_expressions.py
index 208dcab73..0487b90b3 100644
--- a/skyvern/forge/sdk/copilot/composition_browser_expressions.py
+++ b/skyvern/forge/sdk/copilot/composition_browser_expressions.py
@@ -174,6 +174,81 @@ def selector_match_count_expression(css_selector: str) -> str:
return f"(() => {{ try {{ return document.querySelectorAll({sel}).length; }} catch (e) {{ return -1; }}}})()"
+def scout_dynamic_row_evidence_expression(selector: str) -> str:
+ """Capture a bounded, source-page row identity for an exact positional click target.
+
+ Period capture intentionally recognizes English ``Month D, YYYY`` labels only. Keep this browser
+ parser in lockstep with ``_ROW_PERIOD_DATE_RE`` when adding formats; do not add site-specific cases.
+ """
+ sel = json.dumps(selector)
+ return (
+ "(() => {"
+ f" const targetSelector = {sel};"
+ " const text = (value) => String(value || '').replace(/\\s+/g, ' ').trim().slice(0, 500);"
+ " const months = { january: 1, february: 2, march: 3, april: 4, may: 5, june: 6, "
+ "july: 7, august: 8, september: 9, october: 10, november: 11, december: 12 };"
+ " const daysInMonth = (year, month) => month === 2 ? "
+ "(year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0) ? 29 : 28) : "
+ "([4, 6, 9, 11].includes(month) ? 30 : 31);"
+ " const periods = (value) => {"
+ " const found = [];"
+ " const pattern = /\\b(January|February|March|April|May|June|July|August|September|October|November|December)\\s+(0?[1-9]|[12][0-9]|3[01]),\\s+([0-9]{4})\\b/gi;"
+ " for (const match of String(value || '').matchAll(pattern)) {"
+ " const month = months[match[1].toLowerCase()];"
+ " const day = Number(match[2]); const year = Number(match[3]);"
+ " if (year >= 1 && day <= daysInMonth(year, month)) "
+ "found.push(match[3] + '-' + String(month).padStart(2, '0'));"
+ " }"
+ " return found;"
+ " };"
+ " const escapeCss = (value) => window.CSS && CSS.escape ? CSS.escape(value) : "
+ "String(value).replace(/[^A-Za-z0-9_-]/g, (ch) => '\\\\' + ch);"
+ " let target = null;"
+ " let indexed = null;"
+ " try {"
+ " if (/^\\s*(xpath=|\\(?\\/)/.test(targetSelector)) {"
+ " const xpath = targetSelector.replace(/^\\s*xpath=/, '');"
+ " const result = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null);"
+ " target = result ? result.singleNodeValue : null;"
+ " } else {"
+ " indexed = targetSelector.match(/^(.*)\\s*>>\\s*nth=(\\d+)\\s*$/);"
+ " if (indexed) target = document.querySelectorAll(indexed[1].trim())[Number(indexed[2])] || null;"
+ " else target = document.querySelector(targetSelector);"
+ " }"
+ " } catch (e) { return null; }"
+ " if (!target || !target.tagName) return null;"
+ " const tag = target.tagName.toLowerCase();"
+ " const id = target.getAttribute('id') || '';"
+ " const classes = Array.from(target.classList || []).map(String).filter(Boolean);"
+ " let rowSelector = indexed ? indexed[1].trim() : '';"
+ " if (!rowSelector && id) rowSelector = '#' + escapeCss(id);"
+ " else if (!rowSelector && classes.length) "
+ "rowSelector = tag + classes.map((name) => '.' + escapeCss(name)).join('');"
+ " else if (!rowSelector) rowSelector = tag;"
+ " let rows = [];"
+ " try { rows = Array.from(document.querySelectorAll(rowSelector)); } catch (e) { return null; }"
+ " if (rows.length < 2 || rows.length > 100) return null;"
+ " const selectedIndex = rows.indexOf(target);"
+ " if (selectedIndex < 0) return null;"
+ " const rowText = text(target.textContent);"
+ " if (!rowText) return null;"
+ " const rowTextMatchCount = rows.reduce((count, row) => count + (text(row.textContent) === rowText ? 1 : 0), 0);"
+ " const rowPeriods = rows.map((row) => periods(text(row.textContent)));"
+ " const selectedPeriods = rowPeriods[selectedIndex];"
+ " const uniqueSelectedPeriods = Array.from(new Set(selectedPeriods)).sort();"
+ " if (uniqueSelectedPeriods.length > 20) return null;"
+ " const periodMatches = uniqueSelectedPeriods.map((period) => ({"
+ " period,"
+ " selected_row_match_count: selectedPeriods.filter((candidate) => candidate === period).length,"
+ " row_match_count: rowPeriods.filter((candidates) => candidates.includes(period)).length"
+ " }));"
+ " return { target_selector: targetSelector, row_selector: rowSelector, row_text: rowText, "
+ "row_selector_count: rows.length, row_text_match_count: rowTextMatchCount, period_matches: periodMatches, "
+ "selected_index: selectedIndex };"
+ "})()"
+ )
+
+
# Read only the readonly/disabled control-state booleans for a CSS or XPath selector; never reads the
# element's value. An unresolvable selector or non-CSS/XPath engine returns null (UNKNOWN editability).
def scout_control_state_expression(selector: str) -> str:
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index 1419e4b53..dd89b7380 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -57,6 +57,7 @@ if TYPE_CHECKING:
from skyvern.forge.sdk.copilot.completion_criteria_store import CompletionCriteriaTurnState
from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult
from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext
+ from skyvern.forge.sdk.copilot.mcp_adapter import SkyvernOverlayMCPServer
from skyvern.forge.sdk.copilot.output_extraction_plan import FrozenRequestedOutputExtractionCandidate
from skyvern.forge.sdk.copilot.reached_download_target import ReachedDownloadTarget
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
@@ -207,6 +208,12 @@ class AuthorTimeGateAblationEvent:
payload: AuthorTimeGateAblationPayload = field(default_factory=dict)
+class ScoutedEquivalentInput(TypedDict):
+ input_key: str
+ parameter_value: str
+ transform: str
+
+
class ScoutedInputCorrespondence(TypedDict):
input_key: str
matched_literal: str
@@ -214,6 +221,25 @@ class ScoutedInputCorrespondence(TypedDict):
surface: str
transform: str
position: int
+ equivalent_inputs: NotRequired[list[ScoutedEquivalentInput]]
+
+
+class ScoutedDynamicRowPeriodMatch(TypedDict):
+ period: str
+ selected_row_match_count: int
+ row_match_count: int
+
+
+class ScoutedDynamicRowEvidence(TypedDict):
+ source_url: str
+ target_selector: str
+ row_selector: str
+ row_text: str
+ row_selector_count: int
+ row_text_match_count: int
+ period_matches: list[ScoutedDynamicRowPeriodMatch]
+ selected_index: int
+ evidence_fingerprint: str
class ScoutedInteraction(TypedDict):
@@ -224,6 +250,7 @@ class ScoutedInteraction(TypedDict):
# Grounded value-containment witnesses computed at the update_workflow confluence; drive
# generator-owned templated locators. Empty/absent => literal replay.
input_correspondences: NotRequired[list[ScoutedInputCorrespondence]]
+ dynamic_row_evidence: NotRequired[ScoutedDynamicRowEvidence]
typed_value: NotRequired[str]
key: NotRequired[str]
typed_length: NotRequired[int]
@@ -579,6 +606,12 @@ class AgentContext:
# get_by_role(role, name, exact=True) re-anchor resolves to exactly one live element on the source
# page; a non-unique or nameless ambiguous selector leaves this None so synthesis drops the interaction.
pending_scout_reanchor: tuple[str, str, str] | None = None
+ # Source-bound row identity captured before a positional click dispatches. The post-hook consumes it
+ # only for the exact selector/source pair, so navigation cannot transfer the witness to another click.
+ pending_scout_dynamic_row: ScoutedDynamicRowEvidence | None = None
+ # Connected overlay used by bounded pre-click evidence probes; declared so capture code accesses it
+ # directly instead of silently accepting a dynamically attached dependency.
+ discovery_mcp_server: SkyvernOverlayMCPServer | None = None
# Exact secret strings filled into the live browser this turn (passwords,
# call-time-minted OTP codes). Page-readback tool results are exact-string
# scrubbed against this set before being recorded or returned to the model.
diff --git a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
index 574594df9..cd8993de0 100644
--- a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
+++ b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
@@ -47,6 +47,7 @@ from .scouting import (
_actionable_targets_for_result,
_attach_scout_page_summary,
_capture_scout_ambiguity,
+ _capture_scout_dynamic_row,
_capture_scout_role_name,
_capture_scout_source_url,
_clear_pending_browser_interaction_observation,
@@ -56,6 +57,7 @@ from .scouting import (
_mark_pending_browser_interaction_observation,
_maybe_attach_reached_download_target,
_prenav_ambiguity_for_selector,
+ _prenav_dynamic_row_for_selector,
_prenav_role_name_for_selector,
_record_scouted_interaction,
_register_scout_interaction_observation,
@@ -304,6 +306,7 @@ async def _click_pre_hook(
ctx.pending_scout_role_name = None
ctx.pending_scout_click_selector = None
ctx.pending_scout_ambiguous = None
+ ctx.pending_scout_dynamic_row = None
await _capture_scout_source_url(ctx)
deterministic_result = _strip_intent_for_code_only_selector_action(params, ctx, tool_name="click")
if deterministic_result is not None:
@@ -328,6 +331,8 @@ async def _click_pre_hook(
}
await _capture_scout_role_name(ctx, selector)
await _capture_scout_ambiguity(ctx, selector)
+ if _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER:
+ await _capture_scout_dynamic_row(ctx, selector)
return None
@@ -466,6 +471,10 @@ async def _click_post_hook(
ctx.pending_scout_ambiguous = None
pending_reanchor = ctx.pending_scout_reanchor
ctx.pending_scout_reanchor = None
+ pending_dynamic_row = ctx.pending_scout_dynamic_row
+ ctx.pending_scout_dynamic_row = None
+ if _copilot_block_authoring_policy(ctx) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
+ pending_dynamic_row = None
attempted_selector = ctx.pending_scout_click_selector
ctx.pending_scout_click_selector = None
if result.get("ok") and result.get("data"):
@@ -483,6 +492,7 @@ async def _click_post_hook(
if navigated and not (role and accessible_name):
role, accessible_name = _prenav_role_name_for_selector(pending_role_name, selector)
ambiguous = _prenav_ambiguity_for_selector(pending_ambiguous, selector)
+ dynamic_row_evidence = _prenav_dynamic_row_for_selector(pending_dynamic_row, selector, source_url)
if ambiguous:
role, accessible_name = _prenav_role_name_for_selector(pending_reanchor, selector)
result["data"]["effective_target"] = _effective_target_text(selector, role, accessible_name)
@@ -494,6 +504,7 @@ async def _click_post_hook(
role=role,
accessible_name=accessible_name,
ambiguous=ambiguous,
+ dynamic_row_evidence=dynamic_row_evidence,
)
observation_step, page_evidence = await _register_scout_interaction_observation(
ctx, tool_name="click", selector=selector, source_url=source_url, url=url
diff --git a/skyvern/forge/sdk/copilot/tools/scouting.py b/skyvern/forge/sdk/copilot/tools/scouting.py
index af4f8f7ab..4beceb29c 100644
--- a/skyvern/forge/sdk/copilot/tools/scouting.py
+++ b/skyvern/forge/sdk/copilot/tools/scouting.py
@@ -18,10 +18,14 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (
recorded_outcome_from_scout_act_observe_hollow,
)
from skyvern.forge.sdk.copilot.code_block_synthesis import (
+ _is_positional_selector,
+ dynamic_row_evidence_fingerprint,
+ dynamic_row_period_matches_match_selected_row,
locator_selector_literals,
normalized_locator_expr,
normalized_scout_selector,
synthesize_code_block,
+ validated_dynamic_row_period_matches,
)
from skyvern.forge.sdk.copilot.composition_browser_expressions import (
role_name_match_count_expression as _role_name_match_count_expression,
@@ -29,6 +33,9 @@ from skyvern.forge.sdk.copilot.composition_browser_expressions import (
from skyvern.forge.sdk.copilot.composition_browser_expressions import (
scout_accessible_role_name_expression as _scout_accessible_role_name_expression,
)
+from skyvern.forge.sdk.copilot.composition_browser_expressions import (
+ scout_dynamic_row_evidence_expression as _scout_dynamic_row_evidence_expression,
+)
from skyvern.forge.sdk.copilot.composition_browser_expressions import (
selector_match_count_expression as _selector_match_count_expression,
)
@@ -63,6 +70,7 @@ from skyvern.forge.sdk.copilot.result_evidence import (
from skyvern.forge.sdk.copilot.runtime import (
AgentContext,
PendingBrowserInteractionObservation,
+ ScoutedDynamicRowEvidence,
ScoutedInteraction,
resolve_browser_state_for_context,
)
@@ -196,7 +204,7 @@ async def _capture_accessible_role_name(
selector = _selector_text(selector)
if not selector:
return None
- server = getattr(ctx, "discovery_mcp_server", None)
+ server = ctx.discovery_mcp_server
if server is None:
return None
try:
@@ -234,7 +242,7 @@ async def _selector_live_match_count(
selector = _selector_text(selector)
if not selector:
return None
- server = getattr(ctx, "discovery_mcp_server", None)
+ server = ctx.discovery_mcp_server
if server is None:
return None
timeout = _PRE_NAVIGATION_ROLE_NAME_TIMEOUT_SECONDS if timeout_seconds is None else timeout_seconds
@@ -349,6 +357,99 @@ async def _capture_scout_ambiguity(ctx: AgentContext, selector: str | None) -> N
ctx.pending_scout_reanchor = (selector, role, name)
+async def _capture_scout_dynamic_row(ctx: AgentContext, selector: str | None) -> None:
+ ctx.pending_scout_dynamic_row = None
+ if _copilot_block_authoring_policy(ctx) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
+ return
+ selector = _selector_text(selector)
+ source_url = (ctx.pending_scout_source_url or "").strip()
+ server = getattr(ctx, "discovery_mcp_server", None)
+ if not selector or not _is_positional_selector(selector) or not source_url or server is None:
+ return
+ try:
+ result = await asyncio.wait_for(
+ server.call_internal_tool(
+ "skyvern_evaluate",
+ {"expression": _scout_dynamic_row_evidence_expression(selector)},
+ ),
+ timeout=_PRE_NAVIGATION_ROLE_NAME_TIMEOUT_SECONDS,
+ )
+ except Exception:
+ return
+ if not isinstance(result, dict) or not result.get("ok"):
+ return
+ value = (result.get("data") or {}).get("result")
+ if not isinstance(value, dict):
+ return
+ target_selector = value.get("target_selector")
+ row_selector = value.get("row_selector")
+ row_text = value.get("row_text")
+ row_selector_count = value.get("row_selector_count")
+ row_text_match_count = value.get("row_text_match_count")
+ period_matches = value.get("period_matches")
+ validated_period_matches = (
+ validated_dynamic_row_period_matches(period_matches, row_selector_count)
+ if isinstance(row_selector_count, int) and not isinstance(row_selector_count, bool)
+ else None
+ )
+ selected_index = value.get("selected_index")
+ if (
+ target_selector != selector
+ or not isinstance(row_selector, str)
+ or not row_selector.strip()
+ or not isinstance(row_text, str)
+ or not row_text.strip()
+ or isinstance(row_selector_count, bool)
+ or not isinstance(row_selector_count, int)
+ or row_selector_count < 2
+ or row_selector_count > 100
+ or isinstance(row_text_match_count, bool)
+ or not isinstance(row_text_match_count, int)
+ or row_text_match_count < 1
+ or validated_period_matches is None
+ or not dynamic_row_period_matches_match_selected_row(row_text.strip(), validated_period_matches)
+ or isinstance(selected_index, bool)
+ or not isinstance(selected_index, int)
+ or selected_index < 0
+ or selected_index >= row_selector_count
+ ):
+ return
+ ctx.pending_scout_dynamic_row = ScoutedDynamicRowEvidence(
+ source_url=source_url,
+ target_selector=selector,
+ row_selector=row_selector.strip(),
+ row_text=row_text.strip(),
+ row_selector_count=row_selector_count,
+ row_text_match_count=row_text_match_count,
+ period_matches=validated_period_matches,
+ selected_index=selected_index,
+ evidence_fingerprint=dynamic_row_evidence_fingerprint(
+ source_url=source_url,
+ target_selector=selector,
+ row_selector=row_selector.strip(),
+ row_text=row_text.strip(),
+ row_selector_count=row_selector_count,
+ row_text_match_count=row_text_match_count,
+ period_matches=validated_period_matches,
+ selected_index=selected_index,
+ ),
+ )
+
+
+def _prenav_dynamic_row_for_selector(
+ pending: ScoutedDynamicRowEvidence | None,
+ selector: str,
+ source_url: str | None,
+) -> ScoutedDynamicRowEvidence | None:
+ if pending is None:
+ return None
+ if pending["target_selector"] != _selector_text(selector):
+ return None
+ if pending["source_url"] != (source_url or "").strip():
+ return None
+ return pending
+
+
def _prenav_ambiguity_for_selector(pending: tuple[str, bool] | None, selector: str) -> bool:
"""Return the stashed ambiguity verdict only when the recorded selector matches the probed one, so a
navigating click's verdict is never applied to a different element."""
@@ -517,6 +618,7 @@ def _record_scouted_interaction(
credential_field: str = "",
credential_name: str = "",
ambiguous: bool = False,
+ dynamic_row_evidence: ScoutedDynamicRowEvidence | None = None,
) -> None:
selector = _selector_text(selector)
# press_key may be page-level, so it is recorded by key even with no selector; other tools require one.
@@ -563,6 +665,8 @@ def _record_scouted_interaction(
artifact["credential_name"] = credential_name
if ambiguous:
artifact["ambiguous"] = True
+ if dynamic_row_evidence is not None:
+ artifact["dynamic_row_evidence"] = dynamic_row_evidence
interactions = [
item
for item in ctx.scouted_interactions
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index c20fbb8e7..fa0903243 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -114,6 +114,8 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import (
build_input_templated_locator,
build_same_month_file_match_locator,
credential_scout_gap,
+ dynamic_row_evidence_fingerprint,
+ dynamic_row_period_matches_match_selected_row,
freeze_requested_output_extraction_candidate,
grounded_parameter_key_is_safe,
input_correspondences_for_interaction,
@@ -134,6 +136,7 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import (
templated_selection_locator_binding,
uncovered_required_emitted_interactions,
uncovered_rung_records,
+ validated_dynamic_row_period_matches,
)
from skyvern.forge.sdk.copilot.completion_verification import grade_definition_criteria
from skyvern.forge.sdk.copilot.composition_evidence import (
@@ -8644,12 +8647,70 @@ def _locator_provenance_is_self_validating(provenance: Mapping[str, Any]) -> boo
input_holes = provenance.get("holes")
if not isinstance(input_holes, list) or not input_holes:
return False
+ selector = str(provenance.get("selector") or "")
+ row_text = ""
+ if surface == "row_text":
+ source_url = provenance.get("source_url")
+ target_selector = provenance.get("target_selector")
+ row_selector = provenance.get("row_selector")
+ row_text_value = provenance.get("row_text")
+ row_selector_count = provenance.get("row_selector_count")
+ row_text_match_count = provenance.get("row_text_match_count")
+ period_matches = provenance.get("period_matches")
+ validated_period_matches = (
+ validated_dynamic_row_period_matches(period_matches, row_selector_count)
+ if isinstance(row_selector_count, int) and not isinstance(row_selector_count, bool)
+ else None
+ )
+ selected_index = provenance.get("selected_index")
+ evidence_fingerprint = provenance.get("evidence_fingerprint")
+ if (
+ not isinstance(source_url, str)
+ or not source_url.strip()
+ or not isinstance(target_selector, str)
+ or not target_selector.strip()
+ or not isinstance(row_selector, str)
+ or not row_selector.strip()
+ or not isinstance(row_text_value, str)
+ or not row_text_value.strip()
+ or isinstance(row_selector_count, bool)
+ or not isinstance(row_selector_count, int)
+ or row_selector_count < 2
+ or row_selector_count > 100
+ or isinstance(row_text_match_count, bool)
+ or not isinstance(row_text_match_count, int)
+ or row_text_match_count < 1
+ or row_text_match_count > row_selector_count
+ or validated_period_matches is None
+ or not dynamic_row_period_matches_match_selected_row(row_text_value, validated_period_matches)
+ or isinstance(selected_index, bool)
+ or not isinstance(selected_index, int)
+ or selected_index < 0
+ or selected_index >= row_selector_count
+ or not isinstance(evidence_fingerprint, str)
+ or evidence_fingerprint
+ != dynamic_row_evidence_fingerprint(
+ source_url=source_url,
+ target_selector=target_selector,
+ row_selector=row_selector,
+ row_text=row_text_value,
+ row_selector_count=row_selector_count,
+ row_text_match_count=row_text_match_count,
+ period_matches=validated_period_matches,
+ selected_index=selected_index,
+ )
+ ):
+ return False
+ selector = row_selector
+ row_text = row_text_value
recomputed = build_input_templated_locator(
surface=surface,
- selector=str(provenance.get("selector") or ""),
+ selector=selector,
role=str(provenance.get("role") or ""),
name=str(provenance.get("name") or ""),
holes=input_holes,
+ row_text=row_text,
+ period_matches=validated_period_matches if surface == "row_text" and validated_period_matches else (),
)
return recomputed is not None and recomputed == provenance.get("emitted_literal")
if source == SAME_MONTH_FILE_MATCH_PROVENANCE_SOURCE:
@@ -8709,6 +8770,44 @@ def _locator_provenance_is_self_validating(provenance: Mapping[str, Any]) -> boo
return False
+def _public_locator_provenance(provenance_rows: Sequence[Mapping[str, Any]]) -> list[dict[str, Any]]:
+ public_rows: list[dict[str, Any]] = []
+ for provenance in provenance_rows:
+ if provenance.get("surface") != "row_text":
+ public_rows.append(dict(provenance))
+ continue
+ holes = provenance.get("holes")
+ input_keys: set[str] = set()
+ transforms: set[str] = set()
+ if isinstance(holes, list):
+ for hole in holes:
+ if not isinstance(hole, Mapping):
+ continue
+ inputs = [hole]
+ equivalents = hole.get("equivalent_inputs")
+ if isinstance(equivalents, list):
+ inputs.extend(item for item in equivalents if isinstance(item, Mapping))
+ for witness in inputs:
+ key = str(witness.get("input_key") or "")
+ transform = str(witness.get("transform") or "")
+ if key:
+ input_keys.add(key)
+ if transform:
+ transforms.add(transform)
+ source_url = str(provenance.get("source_url") or "")
+ public_rows.append(
+ {
+ "trajectory_index": provenance.get("trajectory_index"),
+ "source": str(provenance.get("source") or ""),
+ "surface": "row_text",
+ "source_origin": url_origin(source_url) if source_url else None,
+ "input_keys": sorted(input_keys),
+ "transforms": sorted(transforms),
+ }
+ )
+ return public_rows
+
+
_PAGE_MUTATION_METHODS = frozenset(
{
"goto",
@@ -11840,7 +11939,7 @@ def _maybe_impose_synthesized_code_block(
"source_trajectory_count": len(scout_trajectory),
"parameter_keys": parameter_reconciliation.parameter_keys,
"credential_parameter_keys": credential_parameter_keys,
- "selector_provenance": diagnostics.locator_provenance,
+ "selector_provenance": _public_locator_provenance(diagnostics.locator_provenance),
"prior_source": prior_source,
}
if extraction_plan is not None:
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index 3d003796e..ead1cf480 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -34,6 +34,7 @@ from skyvern.constants import (
from skyvern.core.script_generations.fuzzy_matcher import match_option_exact_or_stem
from skyvern.errors.errors import TOTPExpiredError, UserDefinedError, filter_to_user_defined_codes
from skyvern.exceptions import (
+ ActionExecutionTimeout,
CaptchaSolveError,
CardNumberInputMismatch,
EmptySelect,
@@ -2963,41 +2964,44 @@ class ActionHandler:
)
actions_result: list[ActionResult] = []
llm_caller = LLMCallerManager.get_llm_caller(task.task_id)
+ execution_timeout_seconds = _resolve_action_execution_timeout(action)
+ execution_timeout_scope: asyncio.Timeout | None = None
try:
- if action.action_type in ActionHandler._handled_action_types:
- invalid_web_action_check = check_for_invalid_web_action(action, page, scraped_page, task, step)
- if invalid_web_action_check:
- actions_result.extend(invalid_web_action_check)
- return actions_result
-
- # do setup before action handler
- if setup := ActionHandler._setup_action_types.get(action.action_type):
- results = await setup(action, page, scraped_page, task, step)
- actions_result.extend(results)
- if results and results[-1] != ActionSuccess:
+ async with asyncio.timeout(execution_timeout_seconds) as execution_timeout_scope:
+ if action.action_type in ActionHandler._handled_action_types:
+ invalid_web_action_check = check_for_invalid_web_action(action, page, scraped_page, task, step)
+ if invalid_web_action_check:
+ actions_result.extend(invalid_web_action_check)
return actions_result
- # do the handler
- handler = ActionHandler._handled_action_types[action.action_type]
- results = await handler(action, page, scraped_page, task, step)
- actions_result.extend(results)
- await app.AGENT_FUNCTION.wait_for_challenge_solver(page=page)
- # do the teardown
- teardown = ActionHandler._teardown_action_types.get(action.action_type)
- if teardown:
- results = await teardown(action, page, scraped_page, task, step)
+ # do setup before action handler
+ if setup := ActionHandler._setup_action_types.get(action.action_type):
+ results = await setup(action, page, scraped_page, task, step)
+ actions_result.extend(results)
+ if results and results[-1] != ActionSuccess:
+ return actions_result
+
+ # do the handler
+ handler = ActionHandler._handled_action_types[action.action_type]
+ results = await handler(action, page, scraped_page, task, step)
actions_result.extend(results)
+ await app.AGENT_FUNCTION.wait_for_challenge_solver(page=page)
+ # do the teardown
+ teardown = ActionHandler._teardown_action_types.get(action.action_type)
+ if teardown:
+ results = await teardown(action, page, scraped_page, task, step)
+ actions_result.extend(results)
- return actions_result
+ return actions_result
- else:
- LOG.error(
- "Unsupported action type in handler",
- action=action,
- type=type(action),
- )
- actions_result.append(ActionFailure(Exception(f"Unsupported action type: {type(action)}")))
- return actions_result
+ else:
+ LOG.error(
+ "Unsupported action type in handler",
+ action=action,
+ type=type(action),
+ )
+ actions_result.append(ActionFailure(Exception(f"Unsupported action type: {type(action)}")))
+ return actions_result
except MissingElement as e:
LOG.info(
"Known exceptions",
@@ -3026,6 +3030,19 @@ class ActionHandler:
exception_message=str(e),
)
actions_result.append(ActionFailure(e))
+ except asyncio.TimeoutError as e:
+ if execution_timeout_scope is not None and execution_timeout_scope.expired():
+ LOG.error(
+ "Action execution exceeded the max duration and was aborted",
+ action=action,
+ timeout_seconds=execution_timeout_seconds,
+ )
+ actions_result.append(
+ ActionFailure(ActionExecutionTimeout(action.action_type, execution_timeout_seconds))
+ )
+ else:
+ LOG.exception("Unhandled exception in action handler", action=action)
+ actions_result.append(ActionFailure(e))
except Exception as e:
LOG.exception("Unhandled exception in action handler", action=action)
actions_result.append(ActionFailure(e))
@@ -3056,6 +3073,14 @@ class ActionHandler:
return actions_result
+def _resolve_action_execution_timeout(action: actions.Action) -> float:
+ base = float(settings.BROWSER_ACTION_MAX_EXECUTION_SECONDS)
+ # WaitAction sleeps action.seconds by design; give it that budget on top of the cap.
+ if isinstance(action, actions.WaitAction):
+ return base + action.seconds
+ return base
+
+
def check_for_invalid_web_action(
action: actions.Action,
page: Page,
diff --git a/tests/unit/test_action_execution_timeout.py b/tests/unit/test_action_execution_timeout.py
new file mode 100644
index 000000000..a4f66a26c
--- /dev/null
+++ b/tests/unit/test_action_execution_timeout.py
@@ -0,0 +1,79 @@
+import asyncio
+from datetime import UTC, datetime
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from skyvern.forge.sdk.models import StepStatus
+from skyvern.webeye.actions.actions import ActionType, ClickAction, WaitAction
+from skyvern.webeye.actions.handler import ActionHandler, _resolve_action_execution_timeout
+from tests.unit.helpers import make_organization, make_step, make_task
+
+
+def _rig() -> tuple:
+ now = datetime.now(UTC)
+ task = make_task(now, make_organization(now))
+ step = make_step(now, task, step_id="step-1", status=StepStatus.created, order=0, output=None)
+ scraped_page = MagicMock(id_to_element_dict={"el": {"id": "el"}})
+ return task, step, scraped_page, MagicMock()
+
+
+def _app_mock() -> MagicMock:
+ app_mock = MagicMock()
+ app_mock.AGENT_FUNCTION.wait_for_challenge_solver = AsyncMock()
+ return app_mock
+
+
+async def test_hung_action_fails_within_bounded_timeout() -> None:
+ task, step, scraped_page, page = _rig()
+
+ async def hung_handler(*args: object, **kwargs: object) -> None:
+ await asyncio.Event().wait()
+
+ action = ClickAction(element_id="el")
+ with (
+ patch("skyvern.webeye.actions.handler.app", _app_mock()),
+ patch.dict(ActionHandler._handled_action_types, {ActionType.CLICK: hung_handler}),
+ patch.dict(ActionHandler._setup_action_types, {}, clear=True),
+ patch.dict(ActionHandler._teardown_action_types, {}, clear=True),
+ patch("skyvern.webeye.actions.handler.settings.BROWSER_ACTION_MAX_EXECUTION_SECONDS", 0.05, create=True),
+ ):
+ results = await asyncio.wait_for(
+ ActionHandler._handle_action(scraped_page=scraped_page, task=task, step=step, page=page, action=action),
+ timeout=2,
+ )
+ assert len(results) == 1
+ assert results[0].success is False
+ assert results[0].exception_type == "ActionExecutionTimeout"
+ assert "timed out" in (results[0].exception_message or "").lower()
+
+
+async def test_handler_raised_timeout_error_keeps_original_exception_type() -> None:
+ task, step, scraped_page, page = _rig()
+
+ async def raising_handler(*args: object, **kwargs: object) -> None:
+ raise TimeoutError("inner playwright-ish timeout")
+
+ action = ClickAction(element_id="el")
+ with (
+ patch("skyvern.webeye.actions.handler.app", _app_mock()),
+ patch.dict(ActionHandler._handled_action_types, {ActionType.CLICK: raising_handler}),
+ patch.dict(ActionHandler._setup_action_types, {}, clear=True),
+ patch.dict(ActionHandler._teardown_action_types, {}, clear=True),
+ patch("skyvern.webeye.actions.handler.settings.BROWSER_ACTION_MAX_EXECUTION_SECONDS", 60, create=True),
+ ):
+ results = await asyncio.wait_for(
+ ActionHandler._handle_action(scraped_page=scraped_page, task=task, step=step, page=page, action=action),
+ timeout=2,
+ )
+ assert len(results) == 1
+ assert results[0].success is False
+ assert results[0].exception_type == "TimeoutError"
+
+
+def test_wait_action_extends_execution_budget() -> None:
+ with patch("skyvern.webeye.actions.handler.settings.BROWSER_ACTION_MAX_EXECUTION_SECONDS", 100, create=True):
+ assert _resolve_action_execution_timeout(WaitAction(seconds=30)) == 130
+ assert _resolve_action_execution_timeout(ClickAction(element_id="el")) == 100
diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.py
index 1dd9bdcad..f9d616e36 100644
--- a/tests/unit/test_copilot_code_block_persist_seam.py
+++ b/tests/unit/test_copilot_code_block_persist_seam.py
@@ -22069,6 +22069,128 @@ def test_input_templated_provenance_rejects_tamper() -> None:
assert not workflow_update_module._locator_provenance_is_self_validating(missing_holes)
+def test_input_templated_provenance_rejects_dropping_distinct_iso_projection() -> None:
+ selector = 'a[data-date="2026-05-01"][href="/statements/2026-05.pdf"]'
+ holes = workflow_update_module.input_correspondences_for_interaction(
+ {"tool_name": "click", "selector": selector},
+ {"billing_start_date": "2026-05-01"},
+ )
+ emitted = workflow_update_module.build_input_templated_locator(
+ surface="selector", selector=selector, role="", name="", holes=holes
+ )
+ provenance = {
+ "source": workflow_update_module.INPUT_TEMPLATED_PROVENANCE_SOURCE,
+ "surface": "selector",
+ "selector": selector,
+ "emitted_literal": emitted,
+ "holes": [dict(hole) for hole in holes],
+ }
+ assert workflow_update_module._locator_provenance_is_self_validating(provenance)
+
+ tampered = dict(provenance)
+ tampered["holes"] = [dict(holes[0])]
+ assert not workflow_update_module._locator_provenance_is_self_validating(tampered)
+
+
+def test_dynamic_selection_row_input_templated_provenance_recomputes_and_rejects_capture_tamper() -> None:
+ selector = "div.statement-row >> nth=2"
+ row_evidence = {
+ "source_url": "https://example.com/statements",
+ "target_selector": selector,
+ "row_selector": "div.statement-row",
+ "row_text": "Statement May 5, 2026",
+ "row_selector_count": 4,
+ "row_text_match_count": 1,
+ "period_matches": [
+ {"period": "2026-05", "selected_row_match_count": 1, "row_match_count": 1},
+ ],
+ "selected_index": 2,
+ }
+ row_evidence["evidence_fingerprint"] = code_block_synthesis_module.dynamic_row_evidence_fingerprint(**row_evidence)
+ interaction = {
+ "tool_name": "click",
+ "selector": selector,
+ "source_url": "https://example.com/statements",
+ "dynamic_row_evidence": row_evidence,
+ }
+ interaction["input_correspondences"] = workflow_update_module.input_correspondences_for_interaction(
+ interaction,
+ {"download_start_date": "2026-05-01", "download_end_date": "2026-05-31"},
+ )
+ synthesized = code_block_synthesis_module.synthesize_code_block([interaction], strict_selectors=True)
+ assert synthesized is not None
+ provenance = next(
+ record for record in synthesized.diagnostics.locator_provenance if record.get("surface") == "row_text"
+ )
+ assert workflow_update_module._locator_provenance_is_self_validating(provenance)
+
+ tampered = dict(provenance)
+ tampered["row_text_match_count"] = 2
+ assert not workflow_update_module._locator_provenance_is_self_validating(tampered)
+
+ tampered_period = json.loads(json.dumps(provenance))
+ tampered_period["period_matches"][0]["row_match_count"] = 2
+ tampered_period["evidence_fingerprint"] = code_block_synthesis_module.dynamic_row_evidence_fingerprint(
+ source_url=tampered_period["source_url"],
+ target_selector=tampered_period["target_selector"],
+ row_selector=tampered_period["row_selector"],
+ row_text=tampered_period["row_text"],
+ row_selector_count=tampered_period["row_selector_count"],
+ row_text_match_count=tampered_period["row_text_match_count"],
+ period_matches=tampered_period["period_matches"],
+ selected_index=tampered_period["selected_index"],
+ )
+ assert not workflow_update_module._locator_provenance_is_self_validating(tampered_period)
+
+
+def test_dynamic_selection_public_provenance_omits_page_text_query_and_credentials() -> None:
+ provenance = {
+ "trajectory_index": 2,
+ "source": workflow_update_module.INPUT_TEMPLATED_PROVENANCE_SOURCE,
+ "surface": "row_text",
+ "source_url": "https://user:[email protected]/statements?token=secret#private",
+ "target_selector": "div.row >> nth=2",
+ "row_selector": "div.row",
+ "row_text": "Customer Alice secret account text May 5, 2026",
+ "row_selector_count": 4,
+ "row_text_match_count": 1,
+ "period_matches": [
+ {"period": "2026-05", "selected_row_match_count": 1, "row_match_count": 1},
+ ],
+ "selected_index": 2,
+ "emitted_literal": "page.locator('div.row')",
+ "holes": [
+ {
+ "input_key": "download_start_date",
+ "matched_literal": "2026-05",
+ "parameter_value": "2026-05-01",
+ "transform": "iso_date_to_year_month",
+ "position": 35,
+ }
+ ],
+ }
+
+ public = workflow_update_module._public_locator_provenance([provenance])
+
+ serialized = json.dumps(public)
+ assert public == [
+ {
+ "trajectory_index": 2,
+ "source": workflow_update_module.INPUT_TEMPLATED_PROVENANCE_SOURCE,
+ "surface": "row_text",
+ "source_origin": "https://example.com",
+ "input_keys": ["download_start_date"],
+ "transforms": ["iso_date_to_year_month"],
+ }
+ ]
+ for secret in ("Alice", "secret", "password", "token", "2026-05-01"):
+ assert secret not in serialized
+
+ tampered = dict(provenance)
+ tampered["source_url"] = "https://other.example.org/statements"
+ assert not workflow_update_module._locator_provenance_is_self_validating(tampered)
+
+
def test_confluence_stamps_and_clears_input_correspondences() -> None:
ctx = _code_only_ctx()
ctx.scout_trajectory = [
diff --git a/tests/unit/test_copilot_code_block_synthesis.py b/tests/unit/test_copilot_code_block_synthesis.py
index 673932775..1ff912c3a 100644
--- a/tests/unit/test_copilot_code_block_synthesis.py
+++ b/tests/unit/test_copilot_code_block_synthesis.py
@@ -9,6 +9,7 @@ import ast
import asyncio
import json
import keyword
+import re
import sys
import textwrap
from dataclasses import replace
@@ -18,6 +19,7 @@ from typing import Any
import pytest
from structlog.testing import capture_logs
+from skyvern.config import settings
from skyvern.forge.sdk.copilot.authoring_parameter_binding import (
_SELECTION_MATCH_BASES,
AuthoringParameterBindingCandidate,
@@ -55,10 +57,12 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import (
_get_by_role_expr,
_get_by_role_expr_strict,
_is_submit_interaction,
+ _strict_period_date_pattern,
build_input_templated_locator,
build_synthesized_artifact_metadata,
code_contains_credential_fill,
credential_scout_gap,
+ dynamic_row_evidence_fingerprint,
input_correspondences_for_interaction,
is_optional_dismissal_only_trajectory,
obligation_finding_reason_code,
@@ -4247,6 +4251,115 @@ def test_input_correspondence_selector_identity_and_month() -> None:
assert by_key["billing_period"]["transform"] == "month_name_to_iso"
+def test_input_correspondence_iso_dates_collapse_to_canonical_period_key_independent_of_order() -> None:
+ interaction = {"tool_name": "click", "selector": "a[href='/statements/100245_2026-05.pdf']"}
+ forward = input_correspondences_for_interaction(
+ interaction,
+ {"download_start_date": "2026-05-01", "download_end_date": "2026-05-31"},
+ )
+ reverse = input_correspondences_for_interaction(
+ interaction,
+ {"download_end_date": "2026-05-31", "download_start_date": "2026-05-01"},
+ )
+
+ assert forward == reverse
+ period = next(correspondence for correspondence in forward if correspondence["matched_literal"] == "2026-05")
+ assert period["input_key"] == "download_end_date"
+ assert period["transform"] == "iso_date_to_year_month"
+ assert period["equivalent_inputs"] == [
+ {
+ "input_key": "download_start_date",
+ "parameter_value": "2026-05-01",
+ "transform": "iso_date_to_year_month",
+ }
+ ]
+
+
[email protected]("value", ["2026-5-01", "2026-02-30", "2026-13-01", "2026-05-01T00:00:00"])
+def test_input_correspondence_iso_date_transform_is_strict(value: str) -> None:
+ assert (
+ input_correspondences_for_interaction(
+ {"tool_name": "click", "selector": "a[href='/statements/2026-05.pdf']"},
+ {"download_start_date": value},
+ )
+ == []
+ )
+
+
+def test_input_correspondence_prefers_exact_iso_date_over_its_period_projection() -> None:
+ assert input_correspondences_for_interaction(
+ {"tool_name": "click", "selector": '[data-range="2026-08-01"]'},
+ {"billing_start_date": "2026-08-01"},
+ ) == [
+ {
+ "input_key": "billing_start_date",
+ "matched_literal": "2026-08-01",
+ "parameter_value": "2026-08-01",
+ "surface": "selector",
+ "transform": "identity",
+ "position": 13,
+ }
+ ]
+
+
+def test_input_correspondence_keeps_distinct_iso_projection_outside_exact_span() -> None:
+ selector = 'a[data-date="2026-05-01"][href="/statements/2026-05.pdf"]'
+
+ correspondences = input_correspondences_for_interaction(
+ {"tool_name": "click", "selector": selector},
+ {"billing_start_date": "2026-05-01"},
+ )
+
+ assert [(hole["matched_literal"], hole["transform"], hole["position"]) for hole in correspondences] == [
+ ("2026-05-01", "identity", 13),
+ ("2026-05", "iso_date_to_year_month", 44),
+ ]
+ expression = build_input_templated_locator(
+ surface="selector", selector=selector, role="", name="", holes=correspondences
+ )
+ assert expression is not None
+ assert "{billing_start_date}" in expression
+ assert "{_scout_iso_date_to_year_month(billing_start_date)}" in expression
+ interaction = {"tool_name": "click", "selector": selector, "source_url": "https://example.com/statements"}
+ interaction["input_correspondences"] = correspondences
+ synthesized = synthesize_code_block([interaction], strict_selectors=True)
+ assert synthesized is not None
+ assert 'data-date="2026-05-01"' not in synthesized.code
+ assert "/statements/2026-05.pdf" not in synthesized.code
+
+
+def test_input_correspondence_rejects_partially_overlapping_matches() -> None:
+ assert (
+ input_correspondences_for_interaction(
+ {"tool_name": "click", "selector": "a[href='/statements/2026-05.pdf']"},
+ {"period": "2026-05", "month_suffix": "05.pdf"},
+ )
+ == []
+ )
+
+
+def test_synthesized_collision_guard_rejects_runtime_period_divergence_before_browser_mutation() -> None:
+ interaction = {
+ "tool_name": "click",
+ "selector": "a[href='/statements/100245_2026-05.pdf']",
+ "source_url": "https://example.com/statements",
+ }
+ interaction["input_correspondences"] = input_correspondences_for_interaction(
+ interaction,
+ {"download_start_date": "2026-05-01", "download_end_date": "2026-05-31"},
+ )
+
+ synthesized = synthesize_code_block([interaction], strict_selectors=True)
+
+ assert synthesized is not None
+ code = synthesized.code
+ guard = "grounded parameters do not resolve to one period"
+ assert guard in code
+ assert code.index(guard) < code.index("await page.goto")
+ assert "_scout_iso_date_to_year_month(download_end_date)" in code
+ assert "_scout_iso_date_to_year_month(download_start_date)" in code
+
+
def test_templated_hole_uses_validated_span_not_earlier_substring() -> None:
# "Widget" also occurs as a non-boundary substring inside "Widgetry"; the hole must template the
# boundary-validated standalone span, not the first find() hit.
@@ -4269,6 +4382,9 @@ def test_templated_hole_uses_validated_span_not_earlier_substring() -> None:
{"page": "100245"},
{"re": "100245"},
{"_scout_month_to_iso": "100245"},
+ {"_scout_iso_date_to_year_month": "100245"},
+ {"_scout_period_month_name": "100245"},
+ {"_scout_period_year": "100245"},
{"account_number": "100245'] , [href"},
],
)
@@ -4392,6 +4508,234 @@ def test_synthesize_unwitnessed_selector_byte_identical() -> None:
assert "_scout_month_to_iso" not in baseline.code
+def _dynamic_row_click(*, source_url: str = "https://example.com/statements") -> dict[str, Any]:
+ selector = "div.statement-row >> nth=2"
+ row_evidence = {
+ "source_url": "https://example.com/statements",
+ "target_selector": selector,
+ "row_selector": "div.statement-row",
+ "row_text": "Statement May 5, 2026",
+ "row_selector_count": 4,
+ "row_text_match_count": 1,
+ "period_matches": [
+ {"period": "2026-05", "selected_row_match_count": 1, "row_match_count": 1},
+ ],
+ "selected_index": 2,
+ }
+ row_evidence["evidence_fingerprint"] = dynamic_row_evidence_fingerprint(**row_evidence)
+ interaction: dict[str, Any] = {
+ "tool_name": "click",
+ "selector": selector,
+ "source_url": source_url,
+ "dynamic_row_evidence": row_evidence,
+ }
+ return interaction
+
+
+def test_dynamic_row_evidence_fingerprint_is_keyed(monkeypatch: pytest.MonkeyPatch) -> None:
+ evidence = _dynamic_row_click()["dynamic_row_evidence"]
+ payload = {key: value for key, value in evidence.items() if key != "evidence_fingerprint"}
+
+ monkeypatch.setattr(settings, "SECRET_KEY", "dynamic-row-key-alpha")
+ fingerprint_a = dynamic_row_evidence_fingerprint(**payload)
+ monkeypatch.setattr(settings, "SECRET_KEY", "dynamic-row-key-beta")
+ fingerprint_b = dynamic_row_evidence_fingerprint(**payload)
+
+ assert len(fingerprint_a) == 64
+ assert len(fingerprint_b) == 64
+ assert fingerprint_a != fingerprint_b
+
+
+def test_valid_unused_dynamic_row_evidence_preserves_generic_positional_synthesis() -> None:
+ interaction = _dynamic_row_click()
+ evidence = interaction["dynamic_row_evidence"]
+ evidence["row_text"] = "Current statement"
+ evidence["row_text_match_count"] = 2
+ evidence["period_matches"] = []
+ evidence["evidence_fingerprint"] = dynamic_row_evidence_fingerprint(
+ **{key: value for key, value in evidence.items() if key != "evidence_fingerprint"}
+ )
+
+ synthesized = synthesize_code_block([interaction], strict_selectors=True)
+ baseline_interaction = {key: value for key, value in interaction.items() if key != "dynamic_row_evidence"}
+ baseline = synthesize_code_block([baseline_interaction], strict_selectors=True)
+
+ assert synthesized is not None
+ assert baseline is not None
+ assert synthesized.code == baseline.code
+ assert synthesized.parameters == baseline.parameters
+ assert synthesized.diagnostics.dropped_interactions == []
+
+
+def test_dynamic_row_period_requires_unique_period_across_candidate_rows() -> None:
+ interaction = _dynamic_row_click()
+ evidence = interaction["dynamic_row_evidence"]
+ evidence["period_matches"][0]["row_match_count"] = 2
+ evidence["evidence_fingerprint"] = dynamic_row_evidence_fingerprint(
+ **{key: value for key, value in evidence.items() if key != "evidence_fingerprint"}
+ )
+
+ assert (
+ input_correspondences_for_interaction(
+ interaction,
+ {"download_start_date": "2026-05-01", "download_end_date": "2026-05-31"},
+ )
+ == []
+ )
+
+
+def test_stamped_dynamic_row_period_license_fails_closed_when_cross_row_count_is_stale() -> None:
+ interaction = _dynamic_row_click()
+ interaction["input_correspondences"] = input_correspondences_for_interaction(
+ interaction,
+ {"download_start_date": "2026-05-01", "download_end_date": "2026-05-31"},
+ )
+ evidence = interaction["dynamic_row_evidence"]
+ evidence["period_matches"][0]["row_match_count"] = 2
+ evidence["evidence_fingerprint"] = dynamic_row_evidence_fingerprint(
+ **{key: value for key, value in evidence.items() if key != "evidence_fingerprint"}
+ )
+
+ synthesized = synthesize_code_block([interaction], strict_selectors=True)
+
+ assert synthesized is not None
+ assert synthesized.diagnostics.emitted_interactions == []
+ assert synthesized.diagnostics.dropped_interactions[0]["reason_code"] == "invalid_dynamic_row_evidence"
+
+
+ ("period", "matching", "non_matching"),
+ [
+ ("2026-05", "Statement mAy 5, 2026", "Statement May 5, 2025"),
+ ("2026-05", "Statement MAY 05, 2026.", "Statement May 00, 2026."),
+ ("2024-02", "Statement February 29, 2024", "Statement February 31, 2024"),
+ ("2026-02", "Statement February 28, 2026", "Statement February 29, 2026"),
+ ("2026-04", "Statement April 30, 2026", "Statement April 31, 2026"),
+ ],
+)
+def test_period_matcher_is_case_insensitive_and_gregorian_valid(period: str, matching: str, non_matching: str) -> None:
+ pattern = _strict_period_date_pattern(period)
+
+ assert pattern is not None
+ assert pattern.search(matching) is not None
+ assert pattern.search(non_matching) is None
+
+
+def test_period_matcher_rejects_year_zero() -> None:
+ assert _strict_period_date_pattern("0000-05") is None
+
+
+def test_emitted_period_matcher_uses_same_case_and_calendar_contract() -> None:
+ interaction = _dynamic_row_click()
+ interaction["input_correspondences"] = input_correspondences_for_interaction(
+ interaction,
+ {"download_start_date": "2026-05-01", "download_end_date": "2026-05-31"},
+ )
+ synthesized = synthesize_code_block([interaction], strict_selectors=True)
+
+ assert synthesized is not None
+ function = next(
+ node
+ for node in ast.parse(textwrap.dedent(synthesized.code)).body
+ if isinstance(node, ast.FunctionDef) and node.name == "_scout_period_date_pattern"
+ )
+ namespace: dict[str, Any] = {"re": re}
+ exec(ast.unparse(function), namespace)
+ matcher = namespace["_scout_period_date_pattern"]
+
+ assert matcher("2026-05").search("Statement mAy 5, 2026") is not None
+ assert matcher("2026-05").search("Statement MAY 05, 2026.") is not None
+ assert matcher("2026-02").search("Statement February 29, 2026") is None
+ assert matcher("2024-02").search("Statement February 29, 2024") is not None
+ assert matcher("2026-04").search("Statement April 31, 2026") is None
+ with pytest.raises(Exception, match="unrecognized grounded period"):
+ matcher("0000-05")
+
+
+def test_dynamic_row_period_parity_accepts_zero_padded_day_and_trailing_punctuation() -> None:
+ interaction = _dynamic_row_click()
+ evidence = interaction["dynamic_row_evidence"]
+ evidence["row_text"] = "Statement mAy 05, 2026."
+ evidence["evidence_fingerprint"] = dynamic_row_evidence_fingerprint(
+ **{key: value for key, value in evidence.items() if key != "evidence_fingerprint"}
+ )
+
+ correspondences = input_correspondences_for_interaction(
+ interaction,
+ {"download_start_date": "2026-05-01", "download_end_date": "2026-05-31"},
+ )
+
+ assert any(item["surface"] == "row_text" for item in correspondences)
+ interaction["input_correspondences"] = correspondences
+ synthesized = synthesize_code_block([interaction], strict_selectors=True)
+ assert synthesized is not None
+ assert ".filter(has_text=_scout_period_date_pattern(" in synthesized.code
+
+
+def test_dynamic_row_period_evidence_emits_direct_count_guarded_locator() -> None:
+ interaction = _dynamic_row_click()
+ interaction["input_correspondences"] = input_correspondences_for_interaction(
+ interaction,
+ {"download_start_date": "2026-05-01", "download_end_date": "2026-05-31"},
+ )
+
+ synthesized = synthesize_code_block([interaction], strict_selectors=True)
+
+ assert synthesized is not None
+ assert "nth=" not in synthesized.code
+ assert 'page.locator("div.statement-row")' in synthesized.code
+ assert synthesized.code.count("def _scout_period_date_pattern(") == 1
+ assert ".filter(has_text=_scout_period_date_pattern(" in synthesized.code
+ assert "_scout_period_month_name" not in synthesized.code
+ assert "_scout_period_year" not in synthesized.code
+ assert ".count() != 1" in synthesized.code
+ assert {parameter["key"] for parameter in synthesized.parameters} == {
+ "download_end_date",
+ "download_start_date",
+ }
+
+
+def test_dynamic_row_period_matcher_does_not_join_month_and_year_from_unrelated_fields() -> None:
+ interaction = _dynamic_row_click()
+ interaction["input_correspondences"] = input_correspondences_for_interaction(
+ interaction,
+ {"download_start_date": "2026-05-01", "download_end_date": "2026-05-31"},
+ )
+ synthesized = synthesize_code_block([interaction], strict_selectors=True)
+
+ assert synthesized is not None
+ # The generated matcher requires one contiguous witnessed date; month/year in separate fields do not qualify.
+ pattern = _strict_period_date_pattern("2027-06")
+ assert pattern is not None
+ assert pattern.search("Month June | account opened 2027") is None
+ assert pattern.search("Statement June 5, 2027") is not None
+
+
+ "mutation",
+ [
+ lambda evidence: evidence.pop("row_text"),
+ lambda evidence: evidence.update(source_url="https://other.example.org/statements"),
+ lambda evidence: evidence.update(target_selector="div.other-row >> nth=2"),
+ ],
+)
+def test_dynamic_row_period_evidence_fails_closed_when_relation_is_missing_or_competing(mutation: Any) -> None:
+ interaction = _dynamic_row_click()
+ mutation(interaction["dynamic_row_evidence"])
+
+ assert (
+ input_correspondences_for_interaction(
+ interaction,
+ {"download_start_date": "2026-05-01", "download_end_date": "2026-05-31"},
+ )
+ == []
+ )
+ synthesized = synthesize_code_block([interaction], strict_selectors=True)
+ assert synthesized is not None
+ assert synthesized.diagnostics.emitted_interactions == []
+ assert synthesized.diagnostics.dropped_interactions[0]["reason_code"] == "invalid_dynamic_row_evidence"
+
+
def test_synthesize_input_purity() -> None:
trajectory = [_witnessed_click()]
snapshot = json.loads(json.dumps(trajectory))
diff --git a/tests/unit/test_copilot_hooks.py b/tests/unit/test_copilot_hooks.py
index e32ffdf48..92b711caa 100644
--- a/tests/unit/test_copilot_hooks.py
+++ b/tests/unit/test_copilot_hooks.py
@@ -14,7 +14,7 @@ from structlog.testing import capture_logs
from skyvern.forge.sdk.copilot import hooks as hooks_module
from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
-from skyvern.forge.sdk.copilot.code_block_synthesis import synthesize_code_block
+from skyvern.forge.sdk.copilot.code_block_synthesis import dynamic_row_evidence_fingerprint, synthesize_code_block
from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
from skyvern.forge.sdk.copilot.enforcement import CopilotGoalSatisfied
from skyvern.forge.sdk.copilot.hooks import CopilotRunHooks
@@ -24,6 +24,10 @@ from skyvern.forge.sdk.copilot.tools import (
_click_pre_hook,
_prenav_ambiguity_for_selector,
)
+from skyvern.forge.sdk.copilot.tools.scouting import (
+ _capture_scout_dynamic_row,
+ _prenav_dynamic_row_for_selector,
+)
from skyvern.forge.sdk.copilot.turn_halt import CopilotTurnHalt, turn_halt_from_blocker_signal
@@ -1037,6 +1041,7 @@ class TestBrowserInteractionObservationHooks:
pending_scout_click_selector=None,
pending_scout_ambiguous=None,
pending_scout_reanchor=None,
+ pending_scout_dynamic_row=None,
discovery_mcp_server=None,
scouted_interactions=[],
scout_trajectory=[],
@@ -1073,6 +1078,7 @@ class TestBrowserInteractionObservationHooks:
pending_scout_click_selector=None,
pending_scout_ambiguous=None,
pending_scout_reanchor=None,
+ pending_scout_dynamic_row=None,
discovery_mcp_server=None,
scouted_interactions=[],
scout_trajectory=[],
@@ -1130,6 +1136,7 @@ class TestScoutedInteractionCapture:
pending_scout_click_selector=None,
pending_scout_ambiguous=None,
pending_scout_reanchor=None,
+ pending_scout_dynamic_row=None,
discovery_mcp_server=None,
browser_session_id=None,
scouted_interactions=[],
@@ -1490,6 +1497,147 @@ class TestScoutedInteractionCapture:
await _capture_scout_ambiguity(ctx, "#unique")
assert ctx.pending_scout_ambiguous is None
+ @pytest.mark.asyncio
+ async def test_capture_scout_dynamic_row_binds_source_selector_and_unique_row_text(self) -> None:
+ selector = "div.statement-row >> nth=2"
+ server = SimpleNamespace(
+ call_internal_tool=AsyncMock(
+ return_value={
+ "ok": True,
+ "data": {
+ "result": {
+ "target_selector": selector,
+ "row_selector": "div.statement-row",
+ "row_text": "Statement May 5, 2026",
+ "row_selector_count": 4,
+ "row_text_match_count": 1,
+ "period_matches": [
+ {"period": "2026-05", "selected_row_match_count": 1, "row_match_count": 1},
+ ],
+ "selected_index": 2,
+ }
+ },
+ }
+ )
+ )
+ ctx = self._ctx(source_url="https://example.com/statements")
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ ctx.discovery_mcp_server = server
+
+ await _capture_scout_dynamic_row(ctx, selector)
+
+ evidence = ctx.pending_scout_dynamic_row
+ assert evidence is not None
+ assert evidence["source_url"] == "https://example.com/statements"
+ assert evidence["target_selector"] == selector
+ assert evidence["row_selector"] == "div.statement-row"
+ assert evidence["row_text_match_count"] == 1
+ assert evidence["period_matches"] == [
+ {"period": "2026-05", "selected_row_match_count": 1, "row_match_count": 1}
+ ]
+ assert len(evidence["evidence_fingerprint"]) == 64
+ expression = server.call_internal_tool.await_args.args[1]["expression"]
+ assert "let rowSelector = indexed ? indexed[1].trim() : ''" in expression
+ assert "else if (!rowSelector && classes.length)" in expression
+ assert "periodMatches" in expression
+ assert "daysInMonth" in expression
+ assert "0?[1-9]" in expression
+ assert "year >= 1" in expression
+ assert _prenav_dynamic_row_for_selector(evidence, selector, evidence["source_url"]) == evidence
+ assert _prenav_dynamic_row_for_selector(evidence, selector, "https://example.com/other") is None
+
+ @pytest.mark.asyncio
+ async def test_capture_scout_dynamic_row_ignores_non_positional_selector(self) -> None:
+ ctx = self._ctx(source_url="https://example.com/statements")
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ ctx.discovery_mcp_server = SimpleNamespace(call_internal_tool=AsyncMock())
+
+ await _capture_scout_dynamic_row(ctx, "#current-statement-row")
+
+ assert ctx.pending_scout_dynamic_row is None
+ ctx.discovery_mcp_server.call_internal_tool.assert_not_awaited()
+
+ @pytest.mark.asyncio
+ async def test_capture_scout_dynamic_row_does_not_evaluate_in_standard_mode(self) -> None:
+ ctx = self._ctx(source_url="https://example.com/statements")
+ ctx.block_authoring_policy = BlockAuthoringPolicy.STANDARD
+ ctx.discovery_mcp_server = SimpleNamespace(call_internal_tool=AsyncMock())
+
+ await _capture_scout_dynamic_row(ctx, "div.statement-row >> nth=2")
+
+ assert ctx.pending_scout_dynamic_row is None
+ ctx.discovery_mcp_server.call_internal_tool.assert_not_awaited()
+
+ @pytest.mark.asyncio
+ async def test_click_cycle_records_bound_dynamic_row_evidence(self) -> None:
+ selector = "div.statement-row >> nth=2"
+ row_evidence = {
+ "source_url": "https://example.com/statements",
+ "target_selector": selector,
+ "row_selector": "div.statement-row",
+ "row_text": "Statement May 5, 2026",
+ "row_selector_count": 4,
+ "row_text_match_count": 1,
+ "period_matches": [
+ {"period": "2026-05", "selected_row_match_count": 1, "row_match_count": 1},
+ ],
+ "selected_index": 2,
+ }
+ row_evidence["evidence_fingerprint"] = dynamic_row_evidence_fingerprint(**row_evidence)
+ ctx = self._ctx(source_url="https://example.com/statements")
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ ctx.pending_scout_dynamic_row = row_evidence
+
+ await _click_post_hook(
+ {"ok": True, "data": {"selector": selector}},
+ {"browser_context": {"url": "https://example.com/detail", "title": "Statement"}},
+ ctx,
+ )
+
+ assert ctx.scouted_interactions[-1]["dynamic_row_evidence"] == row_evidence
+
+ @pytest.mark.asyncio
+ async def test_click_post_hook_discards_dynamic_row_evidence_in_standard_mode(self) -> None:
+ selector = "div.statement-row >> nth=2"
+ ctx = self._ctx(source_url="https://example.com/statements")
+ ctx.block_authoring_policy = BlockAuthoringPolicy.STANDARD
+ ctx.pending_scout_dynamic_row = {
+ "source_url": "https://example.com/statements",
+ "target_selector": selector,
+ "row_selector": "div.statement-row",
+ "row_text": "Statement May 5, 2026",
+ "row_selector_count": 4,
+ "row_text_match_count": 1,
+ "period_matches": [
+ {"period": "2026-05", "selected_row_match_count": 1, "row_match_count": 1},
+ ],
+ "selected_index": 2,
+ "evidence_fingerprint": "not-consumed-in-standard",
+ }
+
+ await _click_post_hook(
+ {"ok": True, "data": {"selector": selector}},
+ {"browser_context": {"url": "https://example.com/detail", "title": "Statement"}},
+ ctx,
+ )
+
+ assert "dynamic_row_evidence" not in ctx.scouted_interactions[-1]
+
+ @pytest.mark.asyncio
+ async def test_click_pre_hook_does_not_capture_dynamic_row_in_standard_mode(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ from skyvern.forge.sdk.copilot.tools import mcp_hooks as mcp_hooks_module
+
+ capture = AsyncMock()
+ monkeypatch.setattr(mcp_hooks_module, "_capture_scout_dynamic_row", capture)
+ ctx = self._ctx(source_url="https://example.com/statements")
+ ctx.block_authoring_policy = BlockAuthoringPolicy.STANDARD
+
+ await _click_pre_hook({"selector": "div.statement-row >> nth=2"}, ctx)
+
+ capture.assert_not_awaited()
+
@pytest.mark.asyncio
async def test_click_cycle_records_scout_ambiguous_multi_match(self) -> None:
ctx = self._ctx_with_count(2)
diff --git a/tests/unit/test_copilot_scout_act_observe.py b/tests/unit/test_copilot_scout_act_observe.py
index 4eccab36d..10e3f770c 100644
--- a/tests/unit/test_copilot_scout_act_observe.py
+++ b/tests/unit/test_copilot_scout_act_observe.py
@@ -119,6 +119,7 @@ def _ctx(*, server: Any = None, source_url: str | None = _SOURCE_URL) -> SimpleN
pending_scout_click_selector=None,
pending_scout_ambiguous=None,
pending_scout_reanchor=None,
+ pending_scout_dynamic_row=None,
discovery_mcp_server=server,
scouted_interactions=[],
scout_trajectory=[],